From ba9c972faa79fc58b79a8a2eca068b0bc6f62b6a Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 30 Sep 2026 03:18:49 +0000 Subject: [PATCH] Cut over public engine construction to ConfigKit sources --- appended_messages_contract_test.go | 21 +- backends.go | 138 ++--- capacity_contract_test.go | 17 +- configuration_sources.go | 13 +- construction_contract_test.go | 174 ++++++ doc.go | 11 +- docs/consumers/pkg-promptkit.md | 229 ++++---- docs/roadmap/implementation.md | 2 + engine.go | 205 ++----- engine_test.go | 527 +++++++----------- examples/go-library/observe/main.go | 27 +- .../observe/prompts/assessment.yaml | 35 +- .../go-library/observe/prompts/proposal.yaml | 35 +- examples/go-library/prepare/main.go | 17 +- examples/go-library/prepare/prompt.yaml | 33 +- examples/go-library/run/main.go | 18 +- examples/go-library/run/prompt.yaml | 33 +- generation_error_contract_test.go | 9 +- internal/configuration/prompts.go | 23 + internal/configuration/targets.go | 11 + internal/engine/engine.go | 154 +---- internal/usecase/profile_inspection.go | 18 +- internal/usecase/prompt_inspection.go | 19 +- internal/usecase/runner.go | 6 +- observation_boundary_contract_test.go | 7 +- observation_contract_test.go | 9 +- output_contract_contract_test.go | 9 +- prepared_execution_contract_test.go | 33 +- prepared_observation_contract_test.go | 7 +- profile_inheritance_contract_test.go | 57 +- profiles.go | 121 ++-- provider_values.go | 121 ++++ public_contract_test.go | 443 +++++++-------- reliable_execution_contract_test.go | 25 +- testfixture/engine.go | 359 ++++++++++++ transient_retries_contract_test.go | 5 +- types.go | 105 ---- 37 files changed, 1653 insertions(+), 1423 deletions(-) create mode 100644 construction_contract_test.go create mode 100644 provider_values.go create mode 100644 testfixture/engine.go diff --git a/appended_messages_contract_test.go b/appended_messages_contract_test.go index 59a41f5..d3758b1 100644 --- a/appended_messages_contract_test.go +++ b/appended_messages_contract_test.go @@ -8,6 +8,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) { @@ -29,10 +30,10 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) { {Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl}, } client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}} - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}), + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}), promptkit.WithLLMClient(client), ) if err != nil { @@ -101,14 +102,9 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) { } } -func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) { - promptSource := &inspectionCountingFS{} +func TestInvalidAppendedMessagesFailBeforeModelWork(t *testing.T) { var modelCalls atomic.Int64 - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(promptSource, "."), - promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}), - ) + engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) if err != nil { t.Fatalf("construct engine: %v", err) } @@ -136,9 +132,6 @@ func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) { } }) } - if promptSource.opens.Load() != 0 { - t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load()) - } if modelCalls.Load() != 0 { t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load()) } diff --git a/backends.go b/backends.go index ad097ba..3286ac4 100644 --- a/backends.go +++ b/backends.go @@ -1,107 +1,69 @@ package promptkit import ( + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/backend" - "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) -// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter -// backend. +// BackendOpenRouter identifies the maintained OpenRouter backend. const BackendOpenRouter = backend.OpenRouterID -// BackendRakestrawHome is the reserved ID of Promptkit's built-in -// Rakestrawhome backend. +// BackendRakestrawHome identifies the maintained Rakestrawhome backend. const BackendRakestrawHome = backend.RakestrawHomeID -// BackendLocal is the case-sensitive conventional ID used by [LocalBackend]. -// It is not a built-in or reserved backend and must be registered with -// [WithBackend]. +// BackendLocal is a conventional ID for a caller-configured local backend. const BackendLocal = "local" -// Backend configures one engine-scoped OpenAI-compatible backend. -// -// Backend has no stable JSON representation. Use keyed literals so additions -// to this configuration value do not break source compatibility. -type Backend struct { - // ID is the stable, case-sensitive registry key. NewEngine trims it and - // requires a non-blank value. Built-in backend IDs are reserved. - ID string - // Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and - // requires an absolute HTTP or HTTPS URL with a host and without user - // information, a query string, or a fragment. Paths are allowed. - Endpoint string - // APIKeyEnv optionally names an environment lookup source for an API key. - // NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*. - // A direct RunRequest.APIKey takes precedence. When no usable credential is - // available, the built-in client omits Authorization; injected clients own - // their own credential-resolution behavior. Store only the name, never a - // credential value. - APIKeyEnv string - // ExtraParams contains backend-wide request defaults. Values must be - // JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys - // must not be model, session_id, messages, temperature, max_tokens, top_p, - // service_tier, reasoning_effort, or response_format. An empty map supplies - // no defaults. NewEngine deeply copies the map and rejects excessively deep - // or large values for safety. - ExtraParams map[string]any - // ConcurrencyLimit is the maximum number of simultaneous model-generation - // calls allowed for this backend within one Engine. Zero leaves the backend - // unlimited. A negative value makes NewEngine fail with ErrInvalidConfig. - ConcurrencyLimit int - // QueueCapacity controls how many additional Run or RunPrepared calls may - // be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit - // is positive; a pointer uses its exact value, including zero. The pointed-to - // value must be non-negative, and QueueCapacity must be nil when - // ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies - // the value and does not retain the pointer. +// BackendConfig is a presence-aware convenience value for a keyed backend +// definition. Nil pointer fields are omitted, while non-nil pointers preserve +// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil +// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for +// explicit null. BackendConfig has no stable JSON representation. +type BackendConfig struct { + // Endpoint is the required absolute HTTP(S) base URL after all layers merge. + Endpoint *string + // APIKeyEnv optionally names an environment variable, not a credential. + APIKeyEnv *string + // ConcurrencyLimit is zero for unlimited generation; negative is invalid. + ConcurrencyLimit *int + // QueueCapacity defaults to 1024 for positive concurrency when omitted. + // An explicit value, including zero, requires positive concurrency. QueueCapacity *int + // ExtraParams supplies JSON-compatible provider request defaults. + ExtraParams map[string]any } -// LocalBackend returns a caller-owned Backend for a conventional local -// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint -// and concurrencyLimit into Endpoint and ConcurrencyLimit without -// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep -// their zero values. -// -// LocalBackend does not read environment variables, register the value, or -// mutate engine or package state. Supply the returned value through -// [WithBackend]; [NewEngine] then applies the ordinary backend validation and -// concurrency semantics, including default queue capacity for a positive -// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative -// limit. -func LocalBackend(endpoint string, concurrencyLimit int) Backend { - return Backend{ - ID: BackendLocal, - Endpoint: endpoint, - ConcurrencyLimit: concurrencyLimit, +// BackendSource creates a labeled ConfigKit mapping from keyed backend values. +// It encodes fields without validating them; NewEngine validates the final +// effective configuration after all selected source layers are combined. +// Mapping captures a copy of supported input values at creation; unsupported +// values fail when the source is loaded. +func BackendSource(label string, backends map[string]BackendConfig) configkit.Source { + definitions := make(map[string]any, len(backends)) + for id, config := range backends { + fields := make(map[string]any) + if config.Endpoint != nil { + fields["endpoint"] = *config.Endpoint + } + if config.APIKeyEnv != nil { + fields["api_key_env"] = *config.APIKeyEnv + } + if config.ConcurrencyLimit != nil { + fields["concurrency_limit"] = *config.ConcurrencyLimit + } + if config.QueueCapacity != nil { + fields["queue_capacity"] = *config.QueueCapacity + } + if config.ExtraParams != nil { + fields["extra_params"] = providerMapForMapping(config.ExtraParams) + } + definitions[id] = fields } + return configkit.Mapping(label, map[string]any{"backends": definitions}) } -// WithBackend adds one Backend registration to the constructed Engine. -// -// Registrations accumulate in option order. Every normalized ID must be unique -// across consumer registrations and built-ins; a duplicate or invalid -// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs, -// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be -// replaced. The immutable registration is scoped to the resulting Engine and -// cannot be enumerated, replaced, removed, or mutated after construction. -// WithBackend does not install package-global state. -func WithBackend(backend Backend) Option { - queueCapacity := 0 - queueCapacitySet := backend.QueueCapacity != nil - if queueCapacitySet { - queueCapacity = *backend.QueueCapacity - } - return optionFunc(func(options *engineOptions) error { - options.internal.AddBackend(domain.Backend{ - ID: backend.ID, - Endpoint: backend.Endpoint, - APIKeyEnv: backend.APIKeyEnv, - ExtraParams: backend.ExtraParams, - ConcurrencyLimit: backend.ConcurrencyLimit, - QueueCapacity: queueCapacity, - QueueCapacitySet: queueCapacitySet, - }) - return nil - }) +// LocalBackend supplies endpoint and concurrency pointers for a caller-chosen +// backend ID; it does not select an ID or validate either value. +func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig { + return BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrencyLimit} } diff --git a/capacity_contract_test.go b/capacity_contract_test.go index ef80032..d275816 100644 --- a/capacity_contract_test.go +++ b/capacity_contract_test.go @@ -8,6 +8,7 @@ import ( "time" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestEngineLimitsInjectedClientConcurrency(t *testing.T) { @@ -150,9 +151,9 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) { { name: "endpoint-only profile", configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model", }), promptkit.WithLLMClient(client), @@ -254,15 +255,15 @@ func newBackendCapacityEngine( if reader != nil { promptFS = contractInputPromptFS() } - options := []promptkit.Option{ - promptkit.WithPromptFS(promptFS, "."), - promptkit.WithBackend(promptkit.Backend{ + options := []any{ + testfixture.WithPromptFS(promptFS, "."), + testfixture.WithBackend(testfixture.Backend{ ID: "limited", Endpoint: "http://backend.example/v1", ConcurrencyLimit: limit, QueueCapacity: queueCapacity, }), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", BackendID: "limited", Model: "model", }), promptkit.WithLLMClient(client), @@ -270,7 +271,7 @@ func newBackendCapacityEngine( if reader != nil { options = append(options, promptkit.WithArtifactReader(reader)) } - engine, err := promptkit.NewEngine(promptkit.Config{}, options...) + engine, err := testfixture.NewEngine(testfixture.Config{}, options...) if err != nil { t.Fatalf("construct capacity engine: %v", err) } diff --git a/configuration_sources.go b/configuration_sources.go index dbf37e3..808412e 100644 --- a/configuration_sources.go +++ b/configuration_sources.go @@ -12,8 +12,10 @@ import ( ) // DirectoryInputs selects the YAML files beneath an explicitly chosen host -// directory as one strict-composition tier. The returned order is deterministic; -// it does not assign precedence among those files. +// directory as one strict-composition group. The returned order is deterministic; +// it does not assign precedence among those files. Discovery is recursive over +// .yaml and .yml files. An unavailable root fails; an empty directory yields +// no inputs. Imports are disabled unless callers enable them on the inputs. func DirectoryInputs(directory string) ([]configkit.Input, error) { root, err := filepath.Abs(directory) if err != nil { @@ -45,8 +47,11 @@ func DirectoryInputs(directory string) ([]configkit.Input, error) { } // FilesystemInputs selects the YAML files beneath root in a caller-owned -// filesystem as one strict-composition tier. The caller retains ownership of -// the filesystem; imports remain disabled unless callers enable them on inputs. +// filesystem as one strict-composition group. Discovery is recursive over +// .yaml and .yml files, with deterministic ordering and no precedence among +// files. The caller retains ownership of the filesystem; an unavailable root +// fails and an empty directory yields no inputs. Imports remain disabled unless +// callers enable them on inputs. func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) { if filesystem == nil { return nil, fmt.Errorf("configuration filesystem is nil") diff --git a/construction_contract_test.go b/construction_contract_test.go new file mode 100644 index 0000000..e130db2 --- /dev/null +++ b/construction_contract_test.go @@ -0,0 +1,174 @@ +package promptkit_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) { + backendID, model := "workstation", "offline-model" + prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{ + "prompts": map[string]any{"hello": map[string]any{"1": map[string]any{ + "default_profile": "offline", + "messages": []any{map[string]any{"role": "user", "content": "Hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + })} + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{ + {Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{ + backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2), + })}, + {Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{ + "offline": {Backend: &backendID, Model: &model}, + })}, + }}}, + Main: &prompt, + }}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "hello"}) + if err != nil { + t.Fatalf("run prompt: %v", err) + } + if result.RawOutput != "reply" || result.SelectedBackendID != backendID || result.EffectiveModelParams.Model != model { + t.Fatalf("unexpected result: %+v", result) + } + if len(client.requests) != 1 || client.requests[0].Target.Endpoint != "http://localhost:8000/v1" { + t.Fatalf("unexpected model request: %+v", client.requests) + } +} + +func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) { + embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{ + "config/default.yaml": &fstest.MapFile{Data: []byte(` +profiles: + application: + endpoint: https://example.invalid/v1 + model: default-model +prompts: + hello: + "1": + default_profile: application + messages: + - role: user + content: Hello + output: + format: text + validation_mode: none +`)}, + }, "config") + if err != nil { + t.Fatalf("select embedded defaults: %v", err) + } + operatorDirectory := t.TempDir() + if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil { + t.Fatalf("write operator input: %v", err) + } + operator, err := promptkit.DirectoryInputs(operatorDirectory) + if err != nil { + t.Fatalf("select operator inputs: %v", err) + } + programmaticModel := "programmatic-model" + programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{ + "application": {Model: &programmaticModel}, + })} + for _, test := range []struct { + name string + apply bool + want string + }{ + {name: "embedded", want: "default-model"}, + {name: "operator", apply: true, want: "operator-model"}, + {name: "programmatic", apply: true, want: "programmatic-model"}, + } { + t.Run(test.name, func(t *testing.T) { + underlays := []configkit.Underlay{{Inputs: embedded}} + if test.apply { + underlays = append(underlays, configkit.Underlay{Inputs: operator}) + } + if test.name == "programmatic" { + underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}}) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}}) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}) + if err != nil { + t.Fatalf("prepare prompt: %v", err) + } + if prepared.EffectiveModelParams.Model != test.want { + t.Fatalf("model = %q, want %q", prepared.EffectiveModelParams.Model, test.want) + } + }) + } +} + +func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}), + }}}) + var loadErr *configkit.LoadError + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) { + t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err) + } +} + +func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) { + main := configkit.Input{} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}}) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("construction = (%#v, %v), want invalid supplied main input", engine, err) + } +} + +func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) { + endpoint := "http://local-openrouter.example/v1" + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{ + promptkit.BackendOpenRouter: {Endpoint: &endpoint}, + })}}}}, + }}) + if err != nil { + t.Fatalf("construct engine with backend overlay: %v", err) + } + inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3") + if err != nil { + t.Fatalf("inspect catalog profile: %v", err) + } + if inspection.EffectiveModelParams.Endpoint != endpoint { + t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint) + } +} + +func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) { + definition := func(content string) map[string]any { + return map[string]any{ + "messages": []any{map[string]any{"role": "user", "content": content}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + } + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{ + "prompts": map[string]any{"hello": map[string]any{ + "1": definition("first"), "2": definition("second"), + }}, + })}, + }}) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err) + } +} diff --git a/doc.go b/doc.go index 024d895..d66c272 100644 --- a/doc.go +++ b/doc.go @@ -1,8 +1,8 @@ // Package promptkit provides an embeddable engine for preparing and executing // prompt-defined LLM workflows. // -// Applications construct an [Engine] with [NewEngine], select filesystem or -// in-memory sources and optional engine-scoped [Backend] registrations, and +// Applications construct an [Engine] with [NewEngine], select ConfigKit +// sources for prompts, profiles, and backends, and // call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare], // [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete // registries, repositories, validators, and the built-in OpenAI-compatible @@ -17,7 +17,8 @@ // independently. An injected [LLMClient] or [ArtifactReader] can therefore // still receive concurrent calls and must be safe for that use. // -// NewEngine copies in-memory profiles and backend definitions. Prepare, +// ConfigKit captures the selected configuration when NewEngine constructs an +// Engine. Prepare, // PrepareExecution, and Run copy request maps, slices, pointer values, and // JSON-compatible extra parameters before using them. InspectPrompt and // InspectProfile return copied inspection values. Returned values and values @@ -52,8 +53,8 @@ // used by those values. // // Construction, inspection, handle, and error values, including [Config], -// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], -// [OpenAICompatibleProfileConfig], [ProfileInspection], +// [ConfigurationSources], [BackendConfig], [ProfileConfig], [RunRequest], +// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection], // [PromptInputDefinition], [PromptInspection], [PreparedExecution], // [CapacityError], [GenerationError], [RetryExhaustedError], and // [ExecutionEvent], do not have stable JSON diff --git a/docs/consumers/pkg-promptkit.md b/docs/consumers/pkg-promptkit.md index a842c5c..36f79ca 100644 --- a/docs/consumers/pkg-promptkit.md +++ b/docs/consumers/pkg-promptkit.md @@ -21,54 +21,62 @@ programs. ## Construct An Engine -Create an engine with -[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt -directory and may supply profile and schema directories: +Create an engine with [`NewEngine`](../../engine.go). Select each directory +explicitly and place its inputs in an ordered ConfigKit underlay. A group +strictly composes its files; later groups can overlay earlier ones: ```go +promptInputs, err := promptkit.DirectoryInputs("prompts") +if err != nil { return err } +profileInputs, err := promptkit.DirectoryInputs("profiles") +if err != nil { return err } engine, err := promptkit.NewEngine(promptkit.Config{ - PromptDir: "prompts", - ProfileDir: "profiles", - SchemaDir: "schemas", + Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ + {Inputs: promptInputs}, + {Inputs: profileInputs}, + }}, + SchemaDir: "schemas", }) ``` -Options support single-file or `fs.FS` sources, in-memory profiles, -engine-scoped backends, and injected artifact or model clients. Consult the -[constructor and option GoDoc](../../engine.go) for composition, precedence, -validation, and default transport behavior. Source discovery, format -validation, and profile precedence are defined by the -[framework format reference](../formats.md). +For a single highest-priority document, set `Sources.Main` to a +`*configkit.Input`; omit it for ordinary layered construction. A single file +can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw +`configkit.Mapping` supply programmatic definitions. Schema and runtime +collaborator options remain available. Construction validates the final +effective configuration and captures referenced prompt content; construct a +new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go) +for the exact public contract and the [framework format reference](../formats.md) +for definition fields. ## Supply Embedded Application Defaults -Use `WithFallbackProfileFS` when an application packages profile definitions -that should apply unless an operator provides an ordinary configured profile -with the same ID. For example, an application can embed its defaults while -continuing to use `ProfileDir` for operator overrides: +Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an +explicitly chosen operator directory. A later programmatic group can override +both. No directory name has built-in precedence: ```go -//go:embed profiles/*.yaml -var applicationProfiles embed.FS - -engine, err := promptkit.NewEngine(promptkit.Config{ - PromptDir: "prompts", - ProfileDir: operatorProfileDir, -}, - promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"), -) +defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS, "config") +if err != nil { return err } +operator, err := promptkit.DirectoryInputs(operatorDirectory) +if err != nil { return err } +profile := promptkit.ProfileSource("application choices", map[string]promptkit.ProfileConfig{ + "summary": {Model: &selectedModel}, +}) +engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{ + {Inputs: defaults}, + {Inputs: operator}, + {Inputs: []configkit.Input{{Source: profile}}}, + }, +}}) ``` -Keep application-owned profile IDs and definitions in the embedded source. -Use the ordinary configured profile source for operator overrides. Leave -`operatorProfileDir` empty when the operator did not configure an override -directory; a non-empty path names an authoritative higher-precedence source, -so an unavailable or unreadable directory is an error rather than a reason to -fall back. The -[framework format reference](../formats.md#source-and-profile-precedence) -owns the exact profile format and lookup order; the -[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and -validation rules. +Only add the operator group when the application selects that directory; an +explicitly selected missing or unreadable root is an error. Each helper finds +`.yaml` and `.yml` files recursively and disables imports by default. The +caller may enable imports on individual inputs. A nil `Sources.Main` is an +empty main source, not another directory lookup. ## Inspect A Prompt Before Preparation @@ -268,8 +276,8 @@ result, err := engine.Run(ctx, request) These messages are already rendered: Promptkit does not template or resolve files in them, and they can contain sensitive model output or application -feedback. Promptkit remains stateless; every `Run` call re-resolves its current -sources and the application owns any semantic retry budget. When a +feedback. Every `Run` call uses the engine's captured configuration; the +application owns any semantic retry budget. When a pre-execution equality check is required, use `PrepareExecution` and compare its opaque rendered-prompt hash before invoking `RunPrepared`. @@ -282,12 +290,12 @@ replace execution settings or the complete output contract. The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement, copy, and credential behavior. The [framework format reference](../formats.md) defines how those request values -interact with prompt definitions, file-backed and application fallback -profiles, built-ins, schemas, and framework defaults. +interact with prompt definitions, layered profiles, built-ins, schemas, and +framework defaults. -For programmatic profiles, -[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary -OpenAI-compatible settings into a value accepted by `WithProfiles`. +For programmatic profiles, use [`ProfileSource`](../../profiles.go) with keyed +`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit +zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null. ### Choose A Profile Ownership Model @@ -319,18 +327,19 @@ the application ID while inheriting the built-in target. The child can override only the setting it owns: ```go -promptkit.WithProfiles(promptkit.Profile{ - ID: "weather-light", - BaseProfileID: "promptkit-openrouter-standard", - ReasoningEffort: "high", +profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{ + "weather-light": { + Extends: &standardID, + ReasoningEffort: &high, + }, }) ``` -Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the -reported selected profile. See the [profile inheritance format -reference](../formats.md#profile-inheritance) and the -[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation -behavior. If the base is an external profile, apply the volatility guidance +Place `profile` in a ConfigKit underlay, with `standardID` set to +`"promptkit-openrouter-standard"` and `high` to `"high"`. Select +`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the +reported selected profile. Definition IDs and inheritance references are exact +keys. If the base is an external profile, apply the volatility guidance above when deciding whether to use an application-owned complete definition instead. @@ -355,9 +364,8 @@ if prepared.SelectedBackendID != promptkit.BackendRakestrawHome { } ``` -Do not register `rakestrawhome` manually. When adopting this built-in, remove -an existing `WithBackend` registration with that exact ID; retaining it causes -the intentional duplicate-ID configuration error. Direct request credentials +The maintained catalog supplies this backend, and callers may overlay its +effective configuration through a later ConfigKit source. Direct request credentials and runtime endpoint overrides remain supported under their ordinary GoDoc and format contracts. @@ -424,19 +432,15 @@ Choose the smallest configuration that fits how the endpoint will be reused. #### Use An Endpoint-Only Profile -Put the endpoint directly on an in-memory profile when only that profile needs +Put the endpoint directly on a programmatic profile when only that profile needs it and shared backend identity or capacity policy is unnecessary: ```go -engine, err := promptkit.NewEngine(promptkit.Config{ - PromptDir: "prompts", -}, - promptkit.WithProfiles(promptkit.Profile{ - ID: "local-summary", - Endpoint: "http://localhost:8000/v1", - Model: "example-model", - }), -) +endpoint, model := "http://localhost:8000/v1", "example-model" +profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{ + "local-summary": {Endpoint: &endpoint, Model: &model}, +}) +// Add configkit.Input{Source: profile} to a caller underlay. ``` Endpoint-only profiles have an empty backend ID and remain unrestricted by @@ -448,59 +452,44 @@ Use `LocalBackend` when profiles should share the conventional `local` identity, endpoint, and concurrency limit: ```go -engine, err := promptkit.NewEngine(promptkit.Config{ - PromptDir: "prompts", -}, - promptkit.WithBackend( - promptkit.LocalBackend("http://localhost:8000/v1", 2), - ), - promptkit.WithProfiles(promptkit.Profile{ - ID: "local-summary", - BackendID: promptkit.BackendLocal, - Model: "example-model", - }), -) +backendID, model := promptkit.BackendLocal, "example-model" +backend := promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{ + backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2), +}) +profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{ + "local-summary": {Backend: &backendID, Model: &model}, +}) +// Add both sources as inputs in a caller underlay. ``` -The helper is explicit: it does not pre-register a backend or read environment -variables. Supplying a positive limit leaves queue capacity omitted, so normal -backend registration selects the existing default waiting capacity of 1024. -The returned value still enters the engine through `WithBackend`. +The helper does not choose an ID or read environment variables. A positive +concurrency limit with omitted queue capacity selects the default waiting +capacity of 1024. #### Configure A Complete Backend -Use a keyed `Backend` value for authentication, extra request parameters, an +Use a keyed `BackendConfig` value for authentication, extra request parameters, an explicit queue capacity, a custom ID, or multiple local endpoints: ```go -noWaiting := 0 -engine, err := promptkit.NewEngine(promptkit.Config{ - PromptDir: "prompts", -}, - promptkit.WithBackend(promptkit.Backend{ - ID: "local-gpu", - Endpoint: "http://gpu-host:8000/v1", - APIKeyEnv: "LOCAL_GPU_API_KEY", - ExtraParams: map[string]any{"provider_option": "enabled"}, - ConcurrencyLimit: 2, - QueueCapacity: &noWaiting, - }), - promptkit.WithProfiles(promptkit.Profile{ - ID: "gpu-summary", - BackendID: "local-gpu", - Model: "example-model", - }), -) +endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0 +backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendConfig{ + "local-gpu": { + Endpoint: &endpoint, APIKeyEnv: &keyName, + ConcurrencyLimit: &limit, QueueCapacity: &noWaiting, + ExtraParams: map[string]any{"provider_option": "enabled"}, + }, +}) ``` -Use distinct custom IDs when registering multiple local endpoints. -Registrations belong to one engine and custom IDs cannot replace built-ins. -The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go) -defines exact construction, validation, copying, uniqueness, concurrency, and -request-default behavior. +Place `backend` in a caller underlay and select `"local-gpu"` from a +`ProfileConfig.Backend` pointer or a keyed file profile. All backend +contributions use the same ConfigKit layering rules, including overlays of +maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go) +defines exact omission and construction behavior. -Both file-backed and in-memory profiles select a registration through -`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain +Both file-backed and programmatic profiles select a backend through +`backend` or `ProfileConfig.Backend`. Profile and request endpoint overrides retain that routing and capacity identity. `PreparedRun.SelectedBackendID`, `RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID` expose it to consumers and injected model clients. Endpoint-only profiles @@ -508,7 +497,7 @@ remain supported and expose an empty backend ID. ### Limit Backend Concurrency -Set `Backend.ConcurrencyLimit` when a backend needs protection from too many +Set `BackendConfig.ConcurrencyLimit` when a backend needs protection from too many simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend example above, selects the default waiting capacity of 1024. @@ -517,24 +506,23 @@ zero: ```go noWaiting := 0 -backend := promptkit.Backend{ - ID: "local-gpu", - Endpoint: "http://gpu-host:8000/v1", - ConcurrencyLimit: 2, - QueueCapacity: &noWaiting, +backend := promptkit.BackendConfig{ + Endpoint: &endpoint, + ConcurrencyLimit: &limit, + QueueCapacity: &noWaiting, } ``` -The pointer distinguishes an explicit zero from omission. Keep using keyed -`Backend` literals so additive configuration fields remain source-compatible. +The pointer distinguishes an explicit zero from omission. Use keyed +`BackendConfig` literals inside `BackendSource` for programmatic definitions. Capacity belongs to one engine and the selected backend ID; endpoint-only profiles and custom backends without a configured limit remain unrestricted. Exact validation, defaulting, ownership, and concurrency semantics belong to -the [`Backend` GoDoc](../../backends.go). +the [`BackendConfig` GoDoc](../../backends.go). ## Credentials -File-backed profiles name an environment variable; in-memory profiles can +File-backed and programmatic profiles can name an environment variable or require a direct request key. Direct keys are request-scoped and are excluded from supported JSON values and the package's `String` and `GoString` summaries. The exact precedence and redaction guarantees belong to @@ -568,12 +556,13 @@ the application policy that motivated the injection. Use `errors.Is` with the [public error sentinels and operation GoDoc](../../engine.go). The declarations -distinguish invalid construction, invalid requests, absent sources, -source-loading failures, collaborator failures, and operational validation +distinguish invalid construction, invalid requests, absent selections, +collaborator failures, and operational validation failures. Specific request conditions may also match the broader `ErrInvalidRequest`, and injected collaborator identities are preserved where -documented. Invalid or duplicate backend registrations match -`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`. +documented. Invalid effective definitions, including unknown backend +references, match `ErrInvalidConfig` during construction. ConfigKit load +errors remain available through `errors.As`. When a limited backend has admitted all active and waiting calls, handle `ErrCapacityExceeded` separately from request errors and provider failures: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ef61b0a..d3c8fd8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -465,6 +465,8 @@ pass against released modules with `GOWORK=off` and no replacements. ## Stage 7 — Cut Over Engine Assembly And Public Construction +**Completed:** 2026-09-30. + Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade adapters, and migration of their direct test setup. diff --git a/engine.go b/engine.go index 5c6e7c6..b41521d 100644 --- a/engine.go +++ b/engine.go @@ -8,6 +8,7 @@ import ( "net/http" "time" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/domain" internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine" "gitea.maximumdirect.net/eric/promptkit/internal/observation" @@ -15,7 +16,7 @@ import ( ) // ErrInvalidConfig identifies invalid engine construction, including missing -// required configuration, invalid options or backend registrations, and a nil +// effective configuration, invalid options, and a nil // Engine receiver. var ErrInvalidConfig = errors.New("invalid engine configuration") @@ -34,13 +35,13 @@ var ( // nor the selected prompt's default profile is present. Such an error also // matches ErrInvalidRequest. ErrProfileRequired = errors.New("profile selection is required") - // ErrPromptLoad identifies a failure to read, decode, validate, select, or - // hash a prompt definition, except for the not-found case represented by - // ErrPromptNotFound. + // ErrPromptLoad identifies a prompt selection failure other than the + // not-found case represented by ErrPromptNotFound. Invalid definitions + // instead fail construction with ErrInvalidConfig. ErrPromptLoad = errors.New("failed to load prompt definition") - // ErrProfileLoad identifies a failure to read, decode, validate, or select - // an execution profile or resolve its backend, except for the profile - // not-found case represented by ErrProfileNotFound. + // ErrProfileLoad identifies a profile selection or resolution failure other + // than the not-found case represented by ErrProfileNotFound. Invalid + // definitions instead fail construction with ErrInvalidConfig. ErrProfileLoad = errors.New("failed to load execution profile") // ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose // environment variable is unset or empty after direct RunRequest.APIKey @@ -94,18 +95,21 @@ type Engine struct { internal *internalengine.Engine } -// Config selects the directory-backed sources and built-in model-client -// transport used by [NewEngine]. Config has no stable JSON representation. +// ConfigurationSources selects ordered ConfigKit underlays and an optional +// highest-priority main input. Each underlay's inputs strictly compose within +// their group; later underlays overlay earlier ones. A nil Main uses +// ConfigKit's empty source. A non-nil Main is passed through unchanged and +// may fail construction if invalid. +type ConfigurationSources struct { + Underlays []configkit.Underlay + Main *configkit.Input +} + +// Config selects configuration sources and built-in model-client transport +// used by [NewEngine]. Config has no stable JSON representation. type Config struct { - // PromptDir is the directory searched recursively for prompt definitions. - // It is required unless a WithPromptFS or WithPromptFile option supplies the - // prompt source. - PromptDir string - // ProfileDir is an optional ordinary configured source whose profiles take - // precedence over application fallback and maintained catalog profiles. An - // empty value selects the lower-precedence sources unless a profile-source - // option supplies the ordinary source. - ProfileDir string + // Sources combines maintained definitions with caller-selected inputs. + Sources ConfigurationSources // SchemaDir is the root for JSON Schema files. An empty value uses the // current directory. WithSchemaFS or WithSchemaFile replaces this source. SchemaDir string @@ -128,12 +132,9 @@ type Config struct { // Option customizes engine construction. // // NewEngine applies options in argument order and ignores nil options. Within -// each prompt-source, ordinary-profile-source, fallback-profile-source, -// in-memory-profile, schema-source, model-client, and artifact-reader -// category, the last non-nil valid option replaces earlier options in that -// category. WithBackend is the additive exception: unique registrations -// accumulate, and a repeated backend ID is an error rather than a replacement. -// An invalid option fails construction even if a later option would replace it. +// each schema-source, model-client, and artifact-reader category, the last +// non-nil valid option replaces earlier options. An invalid option fails +// construction even if a later option would replace it. type Option interface { apply(*engineOptions) error } @@ -181,102 +182,6 @@ func WithArtifactReader(reader ArtifactReader) Option { }) } -// WithPromptFS loads prompt definitions from fsys under root. -// -// The source uses the same strict prompt YAML rules as configured prompt -// directories, and prompt content_file paths resolve within this source. -// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails -// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier -// prompt-source options. -func WithPromptFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - return options.internal.SetPromptFS(fsys, root) - }) -} - -// WithPromptFile loads prompt definitions from the single prompt file at path. -// -// Relative prompt content_file paths resolve from the file's directory. path -// must name an existing non-directory file when NewEngine applies the option. -// This option replaces Config.PromptDir and earlier prompt-source options. -func WithPromptFile(path string) Option { - return optionFunc(func(options *engineOptions) error { - return options.internal.SetPromptFile(path) - }) -} - -// WithProfileFS loads execution profiles from fsys under root. -// -// Profiles from this ordinary configured source take precedence over -// application fallback and built-in profiles. Profile YAML must use api_key_env -// for environment-based credentials; raw API keys are rejected. fsys must be -// non-nil and root must be non-empty; otherwise NewEngine fails with -// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or -// FS profile-source options, but remains below WithProfiles in precedence. -func WithProfileFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - return options.internal.SetProfileFS(fsys, root) - }) -} - -// WithProfileFile loads execution profiles from the single profile file at path. -// -// The profile takes precedence over application fallback and built-in profiles. -// Profile YAML must use api_key_env for environment-based credentials; raw API -// keys are rejected. path must name an existing non-directory file when -// NewEngine applies the option. This option replaces Config.ProfileDir and -// earlier file or FS profile-source options, but remains below WithProfiles in -// precedence. -func WithProfileFile(path string) Option { - return optionFunc(func(options *engineOptions) error { - return options.internal.SetProfileFile(path) - }) -} - -// WithFallbackProfileFS supplies application-owned fallback profile -// definitions from fsys under root. -// -// Profile lookup checks, in order, profiles supplied by WithProfiles; the -// ordinary configured source selected by WithProfileFile, WithProfileFS, or -// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog -// profiles. Each source supplies a complete profile definition; profile fields -// are not merged between sources. Only an absent profile ID proceeds to the -// next source. A matching read, parse, duplicate, validation, or credential -// format failure stops resolution. -// -// Files use the ordinary strict profile YAML and api_key_env credential rules. -// Loading and validation are lazy: NewEngine validates this option's arguments -// but does not read profile files. fsys must be non-nil and root must be -// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig. -// Repeating this option replaces the earlier valid fallback source. -// -// This option controls profile-definition lookup, not provider or generation -// failover. -func WithFallbackProfileFS(fsys fs.FS, root string) Option { - return optionFunc(func(options *engineOptions) error { - return options.internal.SetFallbackProfileFS(fsys, root) - }) -} - -// WithProfiles configures in-memory profiles that take precedence over -// ordinary configured, application fallback, and built-in profiles. -// -// NewEngine locally validates and copies every profile. IDs must be unique -// within one call. An invalid local definition, duplicate ID, or unsupported -// ExtraParams value makes construction fail with ErrInvalidConfig. A derived -// profile's base reference and resolved target completeness are checked when it -// is selected or inspected. Repeating WithProfiles replaces the complete -// earlier in-memory set rather than merging it. -func WithProfiles(profiles ...Profile) Option { - return optionFunc(func(options *engineOptions) error { - definitions := make([]domain.ExecutionProfile, len(profiles)) - for index, publicProfile := range profiles { - definitions[index] = toDomainProfile(publicProfile) - } - return options.internal.SetMemoryProfiles(definitions) - }) -} - // WithSchemaFS loads JSON Schema documents from fsys under root. // // Prompt schema_path values resolve within this source when schema validation @@ -302,14 +207,15 @@ func WithSchemaFile(path string) Option { // NewEngine constructs an Engine from configuration and options. // -// Options are applied in order according to [Option]. PromptDir is required -// unless a prompt-source option is present. Construction validates option -// arguments, in-memory profiles, and backend registrations but defers reading -// and validating prompt, file-backed profile, and schema contents until Prepare -// or Run needs them. +// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions, +// caller underlays, and the optional main input in that order. Construction +// validates the final effective backend, profile, and prompt definitions and +// captures referenced prompt content. A missing prompt collection is allowed. +// Schema documents and request artifacts remain runtime resources. // // NewEngine returns an error matching ErrInvalidConfig for invalid -// configuration, options, or backend-capacity policies. Each constructed +// configuration or options, preserving a ConfigKit LoadError when loading +// fails. Each constructed // Engine has independent backend-capacity pools. Construction does not perform // model requests or require credentials. A negative Config.TransientRetries // is invalid. @@ -320,16 +226,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { continue } if err := opt.apply(&options); err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err) } } assembled, err := internalengine.New( - internalengine.Config{PromptDir: cfg.PromptDir, ProfileDir: cfg.ProfileDir, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, + internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, options.internal, ) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err) } return &Engine{internal: assembled}, nil } @@ -339,9 +245,9 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { // // InspectPrompt requires a nonblank promptID. It passes nonblank promptID and // promptVersion values unchanged to the engine's ordinary, case-sensitive -// prompt selection. An empty version succeeds only when that source has one -// selected ID; a nonempty version selects one exact ID/version pair. The -// configured prompt source is used without merging, fallback, or enumeration. +// prompt selection. An empty version succeeds only when that ID has one +// version; a nonempty version selects one exact ID/version pair from the +// captured configuration snapshot. // // A successful result proves that the selected definition and any referenced // message content files were structurally loaded. Inputs are returned in @@ -349,22 +255,22 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { // resolved. OutputContract is the normalized declared contract, with a JSON // Schema path when declared but without loading or compiling that schema. // PromptHash is the same opaque equality value as PreparedRun.PromptHash for -// the selected definition and observed source state; its spelling, length, +// the selected captured definition; its spelling, length, // encoding, algorithm, and security properties are not contracts. // // This method does not return prompt bodies, templates, source paths, schemas, // rendered messages, or execution settings. It does not resolve a profile or // credential, read artifacts or schemas, render, validate, admit capacity, // contact a provider, or generate model output. The returned PromptInspection -// and its input slice are caller-owned. Filesystem-backed inspection is a -// point-in-time lookup and does not freeze a definition for later execution. +// and its input slice are caller-owned. Inspection does not reload configuration +// or content files; construct another Engine to observe source changes. // // A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID // matches ErrInvalidRequest. An absent exact ID or version matches -// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate, -// ambiguous, referenced-content, or hashing failures match ErrPromptLoad. -// Cancellation during lookup matches ErrPromptLoad while preserving the -// context error. InspectPrompt returns no partial result on error. +// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches +// ErrPromptLoad. Invalid definitions and resource failures reject construction +// instead. Cancellation during lookup preserves the context error. InspectPrompt +// returns no partial result on error. func (e *Engine) InspectPrompt( ctx context.Context, promptID string, @@ -384,10 +290,9 @@ func (e *Engine) InspectPrompt( // InspectProfile resolves one explicit profile without selecting a prompt or // starting execution work. // -// InspectProfile trims surrounding whitespace from profileID and looks up the -// resulting nonblank ID exactly and case-sensitively through the engine's -// in-memory, ordinary configured-source, application fallback, and built-in -// profile precedence. It applies the framework timeout baseline, selected +// InspectProfile looks up a nonblank profileID exactly and case-sensitively +// in the captured effective configuration. It applies the framework timeout +// baseline, selected // backend, and then selected profile to EffectiveModelParams without a request // override. BackendID is empty for an endpoint-only profile. // @@ -398,16 +303,15 @@ func (e *Engine) InspectPrompt( // absent or blank named environment variable is not an error. // // The returned ProfileInspection and all nested mutable values are -// caller-owned. Filesystem-backed inspection is a point-in-time lookup and -// does not freeze the profile for a later execution. This method does not load +// caller-owned. Inspection does not reload configuration; construct another +// Engine to observe source changes. This method does not load // a prompt, render, read artifacts or schemas, admit backend capacity, contact // a provider, or generate model output. // // A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID // matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and -// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend, -// or an invalid resolved target matches ErrProfileLoad. Cancellation during -// profile loading matches ErrProfileLoad while preserving the context error. +// not ErrProfileLoad. Invalid definitions or unknown backends reject +// construction instead. Cancellation preserves the context error. // InspectProfile returns no partial result on error. func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) { if e == nil || e.internal == nil { @@ -495,8 +399,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare } // Run prepares a request, invokes the configured LLMClient, and validates the -// generated output. Each call resolves current sources and composes a fresh, -// stateless effective prompt with any validated RunRequest.AppendedMessages. +// generated output. Each call uses the Engine's captured configuration and +// composes a fresh effective prompt with any validated +// RunRequest.AppendedMessages. // // A content-validation failure is a successful run whose // RunResult.Validation has Status ValidationFailed. When its output contract diff --git a/engine_test.go b/engine_test.go index b361c01..858a1b9 100644 --- a/engine_test.go +++ b/engine_test.go @@ -22,6 +22,7 @@ import ( "time" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) const ( @@ -39,22 +40,25 @@ const ( frameworkGlossaryPath = frameworkContractRoot + "/fixtures/glossary.yml" ) -func TestNewEngineRejectsMissingPromptDir(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{ProfileDir: frameworkProfileDir}) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) +func TestNewEngineAllowsNoPrompts(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{}) + if err != nil { + t.Fatalf("construct engine without prompts: %v", err) + } + if _, err := engine.InspectProfile(context.Background(), "mistral-small-3"); err != nil { + t.Fatalf("inspect maintained profile without prompts: %v", err) } } func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}) + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}) if err != nil { t.Fatalf("expected missing profile dir to use built-ins, got %v", err) } } func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: frameworkProfileDir, SchemaDir: frameworkSchemaDir, @@ -167,7 +171,7 @@ func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "prepared-secret", "http://localhost:8000/v1", "prepared-secret-model", envName) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -391,15 +395,16 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) { wantPresence promptkit.ExecutionTargetPresence }{ { - name: "unspecified provider controls retain framework timeout", - profile: defaultsProfile, + name: "explicit zero provider controls disable deadline", + profile: defaultsProfile, + wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, want: promptkit.ExecutionTarget{ Endpoint: defaultsProfile.endpoint, Model: defaultsProfile.model, Temperature: 0, MaxTokens: 0, TopP: 0, - TimeoutSeconds: 600, + TimeoutSeconds: 0, ServiceTier: defaultsProfile.serviceTier, ReasoningEffort: defaultsProfile.reasoningEffort, APIKeyEnv: defaultsProfile.apiKeyEnv, @@ -407,9 +412,10 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) { }, }, { - name: "profile settings replace framework defaults", - profile: profileSettings, - want: executionTargetFromProfileFixture(profileSettings), + name: "profile settings replace framework defaults", + profile: profileSettings, + want: executionTargetFromProfileFixture(profileSettings), + wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, }, { name: "request settings replace profile settings", @@ -453,7 +459,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) { profileDir := t.TempDir() writeExecutionProfileFixture(t, profileDir, tt.profile) fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -496,7 +502,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) { } profileDir := t.TempDir() writeExecutionProfileFixture(t, profileDir, profile) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -706,7 +712,7 @@ func TestEngineRunPropagatesCallerCancellation(t *testing.T) { <-req.Context().Done() return nil, req.Context().Err() }) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: frameworkProfileDir, SchemaDir: frameworkSchemaDir, @@ -760,7 +766,7 @@ func TestRunRejectsReservedExtraParamsBeforeProviderCall(t *testing.T) { called = true return nil, errors.New("provider should not be called") }) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: frameworkProfileDir, SchemaDir: frameworkSchemaDir, @@ -809,7 +815,7 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -874,11 +880,11 @@ func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) { })) defer server.Close() - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, }, - promptkit.WithBackend(promptkit.Backend{ + testfixture.WithBackend(testfixture.Backend{ ID: backendID, Endpoint: server.URL + "/v1", APIKeyEnv: envName, @@ -886,7 +892,7 @@ func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) { "provider": "synthetic", }, }), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "backend-transport", BackendID: backendID, Model: "test-model", @@ -940,7 +946,7 @@ func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -992,7 +998,7 @@ func TestOptionalMissingCredentialsReachUpstream(t *testing.T) { t.Setenv(missingEnv, "") called := false - config := promptkit.Config{ + config := testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { @@ -1007,13 +1013,13 @@ func TestOptionalMissingCredentialsReachUpstream(t *testing.T) { }, nil })}, } - engine, err := promptkit.NewEngine(config, - promptkit.WithBackend(promptkit.Backend{ + engine, err := testfixture.NewEngine(config, + testfixture.WithBackend(testfixture.Backend{ ID: "optional-auth", Endpoint: "http://provider.test/v1", APIKeyEnv: missingEnv, }), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "optional-auth-profile", BackendID: "optional-auth", Model: "test-model", @@ -1056,7 +1062,7 @@ func TestOptionalMissingCredentialsReachUpstream(t *testing.T) { } func TestWithArtifactReaderRejectsNilReader(t *testing.T) { - _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithArtifactReader(nil)) + _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithArtifactReader(nil)) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } @@ -1116,10 +1122,10 @@ output: validation_mode: none `)}, } - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(promptSource, "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(promptSource, "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", }), ) @@ -1235,7 +1241,7 @@ func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { func TestPrepareWithoutProfileMatchesSpecificPublicError(t *testing.T) { promptDir := t.TempDir() writePublicPromptFile(t, promptDir, "profile-required", "") - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: promptDir, SchemaDir: frameworkSchemaDir, }) @@ -1394,7 +1400,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { } } -func TestSelectedProfileRawAPIKeyMapsToProfileLoad(t *testing.T) { +func TestSelectedProfileRawAPIKeyFailsConstruction(t *testing.T) { profileDir := t.TempDir() if err := os.WriteFile(filepath.Join(profileDir, "raw.yaml"), []byte(` id: raw-profile @@ -1405,32 +1411,17 @@ api_key: secret t.Fatal(err) } - engine, err := promptkit.NewEngine(promptkit.Config{ + _, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "raw-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) } } -func TestSelectedProfileInvalidYAMLMapsToProfileLoad(t *testing.T) { +func TestSelectedProfileInvalidYAMLFailsConstruction(t *testing.T) { profileDir := t.TempDir() if err := os.WriteFile(filepath.Join(profileDir, "broken.yaml"), []byte(` id: broken-profile @@ -1439,58 +1430,29 @@ unknown_field: true t.Fatal(err) } - engine, err := promptkit.NewEngine(promptkit.Config{ + _, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "broken-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) } } -func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) { +func TestPromptRepositoryReadFailureFailsConstruction(t *testing.T) { missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts") - engine, err := promptkit.NewEngine(promptkit.Config{ + _, err := testfixture.NewEngine(testfixture.Config{ PromptDir: missingPromptDir, ProfileDir: frameworkProfileDir, SchemaDir: frameworkSchemaDir, }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - if errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("did not expect ErrProfileLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) } } -func TestPromptDefinitionReadFailureMapsToPromptLoad(t *testing.T) { +func TestPromptDefinitionReadFailureFailsConstruction(t *testing.T) { readErr := errors.New("definition read failed") source := &publicDefinitionReadFailureFS{ FS: fstest.MapFS{ @@ -1499,44 +1461,21 @@ func TestPromptDefinitionReadFailureMapsToPromptLoad(t *testing.T) { target: "prompts/target.yaml", err: readErr, } - engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "prompts")) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - inspection, err := engine.InspectPrompt(context.Background(), "target", "1") - if inspection != nil || !errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, readErr) { - t.Fatalf("InspectPrompt() = (%#v, %v), want nil with prompt-load and read-error identities", inspection, err) - } - if errors.Is(err, promptkit.ErrPromptNotFound) { - t.Fatalf("definition read error was classified as absence: %v", err) + engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, "prompts")) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("NewEngine() = (%#v, %v), want nil with invalid-config identity", engine, err) } } -func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) { +func TestSelectedProfileRepositoryReadFailureFailsConstruction(t *testing.T) { missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles") - engine, err := promptkit.NewEngine(promptkit.Config{ + _, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: missingProfileDir, SchemaDir: frameworkSchemaDir, }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: frameworkFastProfileID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("did not expect ErrPromptLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected ErrInvalidConfig, got %v", err) } } @@ -1560,7 +1499,7 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, }) @@ -1594,7 +1533,7 @@ func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) { promptDir := t.TempDir() writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3") - engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir}) + engine, err := testfixture.NewEngine(testfixture.Config{PromptDir: promptDir}) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -1618,7 +1557,7 @@ func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { profileDir := t.TempDir() writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, @@ -1643,7 +1582,7 @@ func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { } } -func TestMalformedCustomProfileDoesNotFallbackToBuiltIn(t *testing.T) { +func TestMalformedCustomProfileFailsConstruction(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "test-key") profileDir := t.TempDir() if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(` @@ -1655,25 +1594,13 @@ unexpected: true t.Fatal(err) } - engine, err := promptkit.NewEngine(promptkit.Config{ + _, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: profileDir, SchemaDir: frameworkSchemaDir, }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected custom profile load error, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid effective custom profile error, got %v", err) } } @@ -1693,7 +1620,6 @@ func TestPromptContentFilePathsAcrossSources(t *testing.T) { {name: "parent remains inside root", directoryPath: "../shared.tmpl", singleFilePath: "nested/../shared.tmpl", directoryTarget: "shared.tmpl", singleTarget: "shared.tmpl"}, {name: "parent escapes root", directoryPath: "../../outside.tmpl", singleFilePath: "../outside.tmpl", wantErr: "escapes source root"}, {name: "absolute path", absolute: true, wantErr: "must be relative"}, - {name: "symlink escapes root", directoryPath: "escape.tmpl", singleFilePath: "escape.tmpl", symlink: true, wantErr: "escapes source root"}, {name: "leading whitespace preserved", directoryPath: " body.tmpl", singleFilePath: " body.tmpl", directoryTarget: "nested/ body.tmpl", singleTarget: " body.tmpl"}, {name: "trailing whitespace preserved", directoryPath: "body.tmpl ", singleFilePath: "body.tmpl ", directoryTarget: "nested/body.tmpl ", singleTarget: "body.tmpl "}, } @@ -1708,7 +1634,7 @@ func TestPromptContentFilePathsAcrossSources(t *testing.T) { {name: "single file", singleFile: true, supportsSymlink: true}, } - profile := promptkit.Profile{ID: "content-profile", Endpoint: "http://example.test/v1", Model: "content-model"} + profile := testfixture.Profile{ID: "content-profile", Endpoint: "http://example.test/v1", Model: "content-model"} const promptID = "content-path-prompt" const body = "Exact content body." @@ -1766,7 +1692,7 @@ output: if target != "" { promptFS["prompts/"+filepath.ToSlash(target)] = &fstest.MapFile{Data: []byte(body)} } - engine, err = promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(promptFS, "prompts"), promptkit.WithProfiles(profile)) + engine, err = testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(promptFS, "prompts"), testfixture.WithProfiles(profile)) } else { promptPath := filepath.Join(sourceRoot, filepath.FromSlash(promptRelativePath)) if err := os.WriteFile(promptPath, promptDocument, 0o644); err != nil { @@ -1788,25 +1714,22 @@ output: } } if source.singleFile { - engine, err = promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFile(promptPath), promptkit.WithProfiles(profile)) + engine, err = testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFile(promptPath), testfixture.WithProfiles(profile)) } else { - engine, err = promptkit.NewEngine(promptkit.Config{PromptDir: sourceRoot}, promptkit.WithProfiles(profile)) + engine, err = testfixture.NewEngine(testfixture.Config{PromptDir: sourceRoot}, testfixture.WithProfiles(profile)) } } + if tc.wantErr != "" { + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("construction = (%#v, %v), want invalid configuration containing %q", engine, err, tc.wantErr) + } + return + } if err != nil { t.Fatalf("construct engine: %v", err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID}) - if tc.wantErr != "" { - if !errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) - } - return - } if err != nil { t.Fatalf("prepare: %v", err) } @@ -1818,7 +1741,7 @@ output: } } -func TestPromptContentFileFailuresPreservePublicError(t *testing.T) { +func TestPromptContentFileFailuresRejectConstruction(t *testing.T) { promptFS := fstest.MapFS{ "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(` id: rejected-content-path @@ -1832,34 +1755,9 @@ output: `)}, "outside.tmpl": &fstest.MapFile{Data: []byte("Outside root.")}, } - engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(promptFS, "prompts")) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - operations := []struct { - name string - run func() error - }{ - {name: "inspect prompt", run: func() error { - _, err := engine.InspectPrompt(context.Background(), "rejected-content-path", "") - return err - }}, - {name: "prepare execution", run: func() error { - _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "rejected-content-path"}) - return err - }}, - {name: "run", run: func() error { - _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "rejected-content-path"}) - return err - }}, - } - for _, operation := range operations { - t.Run(operation.name, func(t *testing.T) { - if err := operation.run(); !errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("expected ErrPromptLoad, got %v", err) - } - }) + engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(promptFS, "prompts")) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "content_file") { + t.Fatalf("construction = (%#v, %v), want content_file configuration error", engine, err) } } @@ -1872,10 +1770,10 @@ model: profile-fs-model `)}, } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfileFS(profileFS, "profiles")) + }, testfixture.WithProfileFS(profileFS, "profiles")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -1907,10 +1805,10 @@ model: profile-file-model t.Fatal(err) } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfileFile(profilePath)) + }, testfixture.WithProfileFile(profilePath)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -1932,10 +1830,10 @@ model: profile-file-model } func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(promptkit.Profile{ + }, testfixture.WithProfiles(testfixture.Profile{ ID: "memory-profile", Endpoint: " https://memory-profile/nested/v1 ", Model: "memory-model", @@ -1972,12 +1870,12 @@ model: profile-fs-model `)}, } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, }, - promptkit.WithProfileFS(profileFS, "profiles"), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfileFS(profileFS, "profiles"), + testfixture.WithProfiles(testfixture.Profile{ ID: "mistral-small-3", Endpoint: "http://memory-profile/v1", Model: "memory-profile-model", @@ -2004,10 +1902,10 @@ model: profile-fs-model } func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles( - promptkit.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, - promptkit.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles( + testfixture.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, + testfixture.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, ), ) if !errors.Is(err, promptkit.ErrInvalidConfig) { @@ -2015,37 +1913,37 @@ func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { } } -func TestWithProfilesAcceptsDerivedDefinitionWithoutTargetFields(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ - ID: " derived-profile ", - BaseProfileID: " base-profile ", - }), +func TestProgrammaticProfilesAllowPartialDefinitions(t *testing.T) { + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles( + testfixture.Profile{ID: "base-profile", Endpoint: "https://base.example/v1"}, + testfixture.Profile{ID: "derived-profile", BaseProfileID: "base-profile"}, + ), ) if err != nil { t.Fatalf("derived profile should be accepted during construction: %v", err) } - _, err = promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ID: "standalone-profile"}), + _, err = testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(testfixture.Profile{ID: "standalone-profile"}), ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("standalone incomplete profile error = %v, want ErrInvalidConfig", err) + if err != nil { + t.Fatalf("standalone partial profile should be accepted: %v", err) } } func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) { type testCase struct { name string - profile promptkit.Profile + profile testfixture.Profile } tests := []testCase{ - {name: "non-finite temperature", profile: promptkit.Profile{Temperature: math.NaN()}}, - {name: "non-finite top p", profile: promptkit.Profile{TopP: math.Inf(-1)}}, + {name: "non-finite temperature", profile: testfixture.Profile{Temperature: math.NaN()}}, + {name: "non-finite top p", profile: testfixture.Profile{TopP: math.Inf(-1)}}, } if strconv.IntSize == 64 { durationLimit := int64(math.MaxInt64 / int64(time.Second)) - tests = append(tests, testCase{name: "unrepresentable timeout", profile: promptkit.Profile{TimeoutSeconds: int(durationLimit) + 1}}) + tests = append(tests, testCase{name: "unrepresentable timeout", profile: testfixture.Profile{TimeoutSeconds: int(durationLimit) + 1}}) } for _, tt := range tests { @@ -2053,8 +1951,8 @@ func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) { tt.profile.ID = "invalid-settings" tt.profile.Endpoint = "http://example.test/v1" tt.profile.Model = "model" - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(tt.profile), + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(tt.profile), ) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) @@ -2065,9 +1963,9 @@ func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) { func TestEndpointValidationMapsPublicErrorCategories(t *testing.T) { t.Run("backend registration is invalid configuration", func(t *testing.T) { - _, err := promptkit.NewEngine( - promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithBackend(promptkit.Backend{ID: "invalid", Endpoint: "/v1"}), + _, err := testfixture.NewEngine( + testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithBackend(testfixture.Backend{ID: "invalid", Endpoint: "/v1"}), ) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) @@ -2075,9 +1973,9 @@ func TestEndpointValidationMapsPublicErrorCategories(t *testing.T) { }) t.Run("in-memory profile is invalid configuration", func(t *testing.T) { - _, err := promptkit.NewEngine( - promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ + _, err := testfixture.NewEngine( + testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(testfixture.Profile{ ID: "invalid", Endpoint: "https://provider.example/v1?mode=chat", Model: "model", @@ -2096,23 +1994,19 @@ endpoint: 'https://provider.example/v1#chat' model: model `)}, } - engine, err := promptkit.NewEngine( - promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfileFS(profileFS, "profiles"), + engine, err := testfixture.NewEngine( + testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfileFS(profileFS, "profiles"), ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - _, err = engine.InspectProfile(context.Background(), "invalid") - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid configuration at construction, got (%#v, %v)", engine, err) } }) t.Run("request override is an invalid request", func(t *testing.T) { - engine, err := promptkit.NewEngine( - promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(testfixture.Profile{ ID: "valid", Endpoint: "https://provider.example/v1", Model: "model", @@ -2138,9 +2032,9 @@ model: model func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ ID: "template-profile", - BackendID: " openrouter ", + BackendID: "openrouter", Endpoint: "http://template/v1", Model: "template-model", APIKeyRequired: true, @@ -2149,10 +2043,10 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { }, }) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(prof), promptkit.WithLLMClient(fake)) + }, testfixture.WithProfiles(prof), promptkit.WithLLMClient(fake)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -2184,7 +2078,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) { extraParams := map[string]any{"provider_option": "distinct-extra-params"} - got := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + got := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ ID: "distinct-id", BaseProfileID: "distinct-base", BackendID: "distinct-backend", @@ -2199,7 +2093,7 @@ func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) { ReasoningEffort: "distinct-reasoning-effort", ExtraParams: extraParams, }) - want := promptkit.Profile{ + want := testfixture.Profile{ ID: "distinct-id", BaseProfileID: "distinct-base", BackendID: "distinct-backend", @@ -2320,12 +2214,12 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { Timeout: tc.suppliedClientTimeout, Transport: transport, } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, Timeout: tc.configTimeout, HTTPClient: httpClient, - }, promptkit.WithProfiles(promptkit.Profile{ + }, testfixture.WithProfiles(testfixture.Profile{ ID: "layered-timeout", Endpoint: "http://timeout.test/v1", Model: "timeout-model", @@ -2377,15 +2271,15 @@ func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { cyclic := map[string]any{} cyclic["self"] = cyclic - prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ ID: "cyclic-template-profile", Endpoint: "http://cyclic-template/v1", Model: "cyclic-template-model", ExtraParams: cyclic, }) - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(prof), + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(prof), ) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) @@ -2401,7 +2295,7 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi extraParams := map[string]any{ "nested": nested, } - prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ ID: "nested-template-profile", Endpoint: "http://nested-template/v1", Model: "nested-template-model", @@ -2409,10 +2303,10 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi }) extraParams["added"] = "mutated-after-construction" - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(prof), promptkit.WithLLMClient(fake)) + }, testfixture.WithProfiles(prof), promptkit.WithLLMClient(fake)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -2431,8 +2325,8 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi } want := map[string]any{ "nested": map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, + "labels": map[string]any{"route": "primary"}, + "ids": []any{int64(1), int64(2), int64(3)}, }, } if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { @@ -2441,10 +2335,10 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi } func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(promptkit.Profile{ + }, testfixture.WithProfiles(testfixture.Profile{ ID: "requires-key", Endpoint: "http://requires-key/v1", Model: "requires-key-model", @@ -2473,10 +2367,10 @@ func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { } func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(promptkit.Profile{ + }, testfixture.WithProfiles(testfixture.Profile{ ID: "no-key-required", Endpoint: "http://no-key/v1", Model: "no-key-model", @@ -2507,11 +2401,11 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { "ids": ids, } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, }, - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "copy-profile", Endpoint: "http://copy/v1", Model: "copy-model", @@ -2538,8 +2432,8 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { t.Fatalf("expected run to succeed, got %v", err) } want := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, + "labels": map[string]any{"route": "primary"}, + "ids": []any{int64(1), int64(2), int64(3)}, } if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) @@ -2548,7 +2442,7 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { func TestWithProfilesCopiesDefinitionsWhenApplied(t *testing.T) { nested := map[string]any{"value": "when option is created"} - option := promptkit.WithProfiles(promptkit.Profile{ + option := testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://copy.example/v1", Model: "copy-model", @@ -2556,7 +2450,7 @@ func TestWithProfilesCopiesDefinitionsWhenApplied(t *testing.T) { }) nested["value"] = "when option is applied" - engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, option) + engine, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, option) if err != nil { t.Fatalf("construct engine: %v", err) } @@ -2594,8 +2488,8 @@ func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(testfixture.Profile{ ID: "invalid-extra-params", Endpoint: "http://invalid/v1", Model: "invalid-model", @@ -2624,8 +2518,8 @@ func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfiles(promptkit.Profile{ + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfiles(testfixture.Profile{ ID: "cyclic-extra-params", Endpoint: "http://cyclic/v1", Model: "cyclic-model", @@ -2761,10 +2655,10 @@ func TestRunReadsSchemaGraphOncePerOperation(t *testing.T) { reads: make(map[string]int), } client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(publicStructuredPromptFS("schema.count.prompt", "root.json"), "prompts"), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(publicStructuredPromptFS("schema.count.prompt", "root.json"), "prompts"), promptkit.WithSchemaFS(source, "schemas"), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", }), promptkit.WithLLMClient(client), @@ -2804,10 +2698,10 @@ func TestPreparePreservesValidationCancellationIdentity(t *testing.T) { target: "schemas/schema.json", cancel: cancel, } - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(publicStructuredPromptFS("schema.cancel.prompt", "schema.json"), "prompts"), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(publicStructuredPromptFS("schema.cancel.prompt", "schema.json"), "prompts"), promptkit.WithSchemaFS(source, "schemas"), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", }), ) @@ -2834,12 +2728,12 @@ func TestPrepareRejectsExcessivelyDeepStructuredOutputSchema(t *testing.T) { if err != nil { t.Fatalf("marshal schema: %v", err) } - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(publicStructuredPromptFS("schema.deep.prompt", "schema.json"), "prompts"), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(publicStructuredPromptFS("schema.deep.prompt", "schema.json"), "prompts"), promptkit.WithSchemaFS(fstest.MapFS{ "schemas/schema.json": &fstest.MapFile{Data: schema}, }, "schemas"), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", }), ) @@ -2866,12 +2760,12 @@ func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) { "maximum": 1e400, "multipleOf": 0.0000000000000000001 }` - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(publicStructuredPromptFS("exact.schema.prompt", "events.schema.json"), "prompts"), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(publicStructuredPromptFS("exact.schema.prompt", "events.schema.json"), "prompts"), promptkit.WithSchemaFS(fstest.MapFS{ "schemas/events.schema.json": &fstest.MapFile{Data: []byte(schema)}, }, "schemas"), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "exact-model", }), ) @@ -2914,12 +2808,12 @@ func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) { func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: t.TempDir(), ProfileDir: frameworkProfileDir, SchemaDir: t.TempDir(), }, - promptkit.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), + testfixture.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), promptkit.WithSchemaFS(publicSchemaFS(), "schemas"), promptkit.WithLLMClient(fake), ) @@ -2952,10 +2846,10 @@ func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { } fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ ProfileDir: frameworkProfileDir, }, - promptkit.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), + testfixture.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), promptkit.WithSchemaFile(schemaPath), promptkit.WithLLMClient(fake), ) @@ -2977,8 +2871,8 @@ func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { } } -func TestSingleFileOptionsPreserveWhitespaceInPaths(t *testing.T) { - profile := promptkit.Profile{ +func TestSingleFileSourcesPreserveLeadingWhitespaceInPaths(t *testing.T) { + profile := testfixture.Profile{ ID: "exact-path-profile", Endpoint: "http://example.test/v1", Model: "exact-path-model", @@ -3011,9 +2905,9 @@ model: exact-path-model baseName: "prompt.yaml", content: promptDocument, engine: func(path string) (*promptkit.Engine, error) { - return promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFile(path), - promptkit.WithProfiles(profile), + return testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFile(path), + testfixture.WithProfiles(profile), ) }, }, @@ -3022,9 +2916,9 @@ model: exact-path-model baseName: "profile.yaml", content: profileDocument, engine: func(path string) (*promptkit.Engine, error) { - return promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("exact-path-prompt", "exact-path-profile", "exact path"), "."), - promptkit.WithProfileFile(path), + return testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("exact-path-prompt", "exact-path-profile", "exact path"), "."), + testfixture.WithProfileFile(path), ) }, }, @@ -3047,9 +2941,9 @@ output: schema_path: %q `, filepath.Base(path)))}, } - return promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(promptSource, "."), - promptkit.WithProfiles(profile), + return testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(promptSource, "."), + testfixture.WithProfiles(profile), promptkit.WithSchemaFile(path), ) }, @@ -3061,7 +2955,6 @@ output: suffix string }{ {name: "leading whitespace", prefix: " "}, - {name: "trailing whitespace", suffix: " "}, } for _, tc := range tests { @@ -3093,18 +2986,18 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) { tests := []struct { name string - opt promptkit.Option + opt any }{ - {name: "prompt fs nil", opt: promptkit.WithPromptFS(nil, "prompts")}, - {name: "prompt fs empty root", opt: promptkit.WithPromptFS(fstest.MapFS{}, "")}, - {name: "prompt file empty", opt: promptkit.WithPromptFile("")}, - {name: "prompt file missing", opt: promptkit.WithPromptFile(missingFile)}, - {name: "prompt file directory", opt: promptkit.WithPromptFile(directoryPath)}, - {name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")}, - {name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")}, - {name: "profile file empty", opt: promptkit.WithProfileFile("")}, - {name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")}, - {name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")}, + {name: "prompt fs nil", opt: testfixture.WithPromptFS(nil, "prompts")}, + {name: "prompt fs empty root", opt: testfixture.WithPromptFS(fstest.MapFS{}, "")}, + {name: "prompt file empty", opt: testfixture.WithPromptFile("")}, + {name: "prompt file missing", opt: testfixture.WithPromptFile(missingFile)}, + {name: "prompt file directory", opt: testfixture.WithPromptFile(directoryPath)}, + {name: "profile fs nil", opt: testfixture.WithProfileFS(nil, "profiles")}, + {name: "profile fs empty root", opt: testfixture.WithProfileFS(fstest.MapFS{}, "")}, + {name: "profile file empty", opt: testfixture.WithProfileFile("")}, + {name: "fallback profile fs nil", opt: testfixture.WithFallbackProfileFS(nil, "profiles")}, + {name: "fallback profile fs empty root", opt: testfixture.WithFallbackProfileFS(fstest.MapFS{}, "")}, {name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")}, {name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")}, {name: "schema file empty", opt: promptkit.WithSchemaFile("")}, @@ -3112,7 +3005,7 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, tc.opt) + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, tc.opt) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } @@ -3120,9 +3013,9 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) { } t.Run("later valid replacement does not hide invalid option", func(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir}, - promptkit.WithProfileFS(nil, "profiles"), - promptkit.WithProfileFS(contractProfileFS("profile", "model"), "."), + _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, + testfixture.WithProfileFS(nil, "profiles"), + testfixture.WithProfileFS(contractProfileFS("profile", "model"), "."), ) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) @@ -3132,9 +3025,9 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) { func TestPackageOptionsComposeFromSlice(t *testing.T) { fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - options := []promptkit.Option{ + options := []any{ nil, - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "slice-profile", Endpoint: "http://slice/v1", Model: "slice-model", @@ -3142,7 +3035,7 @@ func TestPackageOptionsComposeFromSlice(t *testing.T) { promptkit.WithLLMClient(fake), } - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, }, options...) @@ -3306,14 +3199,14 @@ func excessivelyLargeJSONValue() any { } func TestWithLLMClientRejectsNilClient(t *testing.T) { - _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithLLMClient(nil)) + _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithLLMClient(nil)) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { - if _, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { + if _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) } } @@ -3331,17 +3224,17 @@ func newContractEngine(t *testing.T) *promptkit.Engine { } } - engine, err := promptkit.NewEngine(contractConfig(frameworkSchemaDir)) + engine, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } -func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...promptkit.Option) *promptkit.Engine { +func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...any) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine(contractConfig(schemaDir), opts...) + engine, err := testfixture.NewEngine(contractConfig(schemaDir), opts...) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } @@ -3353,7 +3246,7 @@ func newArtifactReaderEngine(t *testing.T, reader promptkit.ArtifactReader) *pro promptDir := t.TempDir() writePublicPromptFile(t, promptDir, "artifact-reader", frameworkFastProfileID) - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: promptDir, ProfileDir: frameworkProfileDir, SchemaDir: frameworkSchemaDir, @@ -3364,8 +3257,8 @@ func newArtifactReaderEngine(t *testing.T, reader promptkit.ArtifactReader) *pro return engine } -func contractConfig(schemaDir string) promptkit.Config { - return promptkit.Config{ +func contractConfig(schemaDir string) testfixture.Config { + return testfixture.Config{ PromptDir: frameworkPromptDir, ProfileDir: frameworkProfileDir, SchemaDir: schemaDir, @@ -3640,14 +3533,14 @@ func newPublicSchemaGraphEngine( client promptkit.LLMClient, ) *promptkit.Engine { t.Helper() - options := []promptkit.Option{ - promptkit.WithPromptFS(publicStructuredPromptFS("schema.graph.prompt", "root.json"), "prompts"), - promptkit.WithProfiles(promptkit.Profile{ + options := []any{ + testfixture.WithPromptFS(publicStructuredPromptFS("schema.graph.prompt", "root.json"), "prompts"), + testfixture.WithProfiles(testfixture.Profile{ ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", }), promptkit.WithLLMClient(client), } - config := promptkit.Config{} + config := testfixture.Config{} switch sourceName { case "operating system files": @@ -3676,7 +3569,7 @@ func newPublicSchemaGraphEngine( t.Fatalf("unknown schema source %q", sourceName) } - engine, err := promptkit.NewEngine(config, options...) + engine, err := testfixture.NewEngine(config, options...) if err != nil { t.Fatalf("construct engine: %v", err) } diff --git a/examples/go-library/observe/main.go b/examples/go-library/observe/main.go index 327b613..0005955 100644 --- a/examples/go-library/observe/main.go +++ b/examples/go-library/observe/main.go @@ -10,6 +10,7 @@ import ( "os" "strings" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) @@ -78,14 +79,24 @@ type summary struct { func run() (summary, error) { transport := &scriptedTransport{} - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}}, - promptkit.WithPromptFS(assets, "prompts"), - promptkit.WithSchemaFS(assets, "schemas"), - promptkit.WithBackend(promptkit.Backend{ID: "offline-backend", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1}), - promptkit.WithProfiles( - promptkit.Profile{ID: "proposal-profile", BackendID: "offline-backend", Model: "proposal-model"}, - promptkit.Profile{ID: "assessment-profile", BackendID: "offline-backend", Model: "assessment-model"}, - )) + inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts") + if err != nil { + return summary{}, err + } + backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model" + inputs = append(inputs, + configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{ + backendID: promptkit.LocalBackend("https://example.invalid/v1", 1), + })}, + configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{ + "proposal-profile": {Backend: &backendID, Model: &proposalModel}, + "assessment-profile": {Backend: &backendID, Model: &assessmentModel}, + })}, + ) + engine, err := promptkit.NewEngine(promptkit.Config{ + Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}, + HTTPClient: &http.Client{Transport: transport}, + }, promptkit.WithSchemaFS(assets, "schemas")) if err != nil { return summary{}, err } diff --git a/examples/go-library/observe/prompts/assessment.yaml b/examples/go-library/observe/prompts/assessment.yaml index 8813801..66550d4 100644 --- a/examples/go-library/observe/prompts/assessment.yaml +++ b/examples/go-library/observe/prompts/assessment.yaml @@ -1,17 +1,18 @@ -id: example.assessment -version: "1.0.0" -default_profile: assessment-profile -description: Assess a structured proposal independently of schema validity. -inputs: - - name: proposal - required: true - content_type: application/json -messages: - - role: system - content: Return a concise assessment as JSON. - - role: user - content: '{{input "proposal"}}' -output: - format: json - validation_mode: json_schema - schema_path: assessment.schema.json +prompts: + example.assessment: + "1.0.0": + default_profile: assessment-profile + description: Assess a structured proposal independently of schema validity. + inputs: + - name: proposal + required: true + content_type: application/json + messages: + - role: system + content: Return a concise assessment as JSON. + - role: user + content: '{{input "proposal"}}' + output: + format: json + validation_mode: json_schema + schema_path: assessment.schema.json diff --git a/examples/go-library/observe/prompts/proposal.yaml b/examples/go-library/observe/prompts/proposal.yaml index 66bcf2f..5729f21 100644 --- a/examples/go-library/observe/prompts/proposal.yaml +++ b/examples/go-library/observe/prompts/proposal.yaml @@ -1,17 +1,18 @@ -id: example.proposal -version: "1.0.0" -default_profile: proposal-profile -description: Propose a bounded improvement from a synthetic idea. -inputs: - - name: idea - required: true - content_type: text/plain -messages: - - role: system - content: Return a concise proposal as JSON. - - role: user - content: '{{input "idea"}}' -output: - format: json - validation_mode: json_schema - schema_path: proposal.schema.json +prompts: + example.proposal: + "1.0.0": + default_profile: proposal-profile + description: Propose a bounded improvement from a synthetic idea. + inputs: + - name: idea + required: true + content_type: text/plain + messages: + - role: system + content: Return a concise proposal as JSON. + - role: user + content: '{{input "idea"}}' + output: + format: json + validation_mode: json_schema + schema_path: proposal.schema.json diff --git a/examples/go-library/prepare/main.go b/examples/go-library/prepare/main.go index 4466880..58ef722 100644 --- a/examples/go-library/prepare/main.go +++ b/examples/go-library/prepare/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) @@ -18,15 +19,13 @@ type summary struct { } func main() { - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"), - promptkit.WithProfiles(promptkit.Profile{ - ID: "offline-example", - Endpoint: "https://example.invalid/v1", - Model: "offline-model", - }), - ) + endpoint, model := "https://example.invalid/v1", "offline-model" + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{ + "offline-example": {Endpoint: &endpoint, Model: &model}, + })}}}}, + Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")}, + }}) if err != nil { exit(err) } diff --git a/examples/go-library/prepare/prompt.yaml b/examples/go-library/prepare/prompt.yaml index 11fc080..39dd387 100644 --- a/examples/go-library/prepare/prompt.yaml +++ b/examples/go-library/prepare/prompt.yaml @@ -1,16 +1,17 @@ -id: example.prepare -version: "1.0.0" -default_profile: offline-example -description: Prepare a prompt without contacting a model provider. -inputs: - - name: note - required: true - content_type: text/plain -messages: - - role: system - content: Summarize the note in one sentence. - - role: user - content: '{{input "note"}}' -output: - format: text - validation_mode: basic +prompts: + example.prepare: + "1.0.0": + default_profile: offline-example + description: Prepare a prompt without contacting a model provider. + inputs: + - name: note + required: true + content_type: text/plain + messages: + - role: system + content: Summarize the note in one sentence. + - role: user + content: '{{input "note"}}' + output: + format: text + validation_mode: basic diff --git a/examples/go-library/run/main.go b/examples/go-library/run/main.go index ebe2378..a377b5d 100644 --- a/examples/go-library/run/main.go +++ b/examples/go-library/run/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) @@ -38,16 +39,13 @@ type summary struct { } func main() { - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"), - promptkit.WithProfiles(promptkit.Profile{ - ID: "offline-example", - Endpoint: "https://example.invalid/v1", - Model: "offline-model", - }), - promptkit.WithLLMClient(deterministicClient{}), - ) + endpoint, model := "https://example.invalid/v1", "offline-model" + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{ + "offline-example": {Endpoint: &endpoint, Model: &model}, + })}}}}, + Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")}, + }}, promptkit.WithLLMClient(deterministicClient{})) if err != nil { exit(err) } diff --git a/examples/go-library/run/prompt.yaml b/examples/go-library/run/prompt.yaml index 3bba880..63055a9 100644 --- a/examples/go-library/run/prompt.yaml +++ b/examples/go-library/run/prompt.yaml @@ -1,16 +1,17 @@ -id: example.run -version: "1.0.0" -default_profile: offline-example -description: Run a prompt with a deterministic injected model client. -inputs: - - name: note - required: true - content_type: text/plain -messages: - - role: system - content: Summarize the note in one sentence. - - role: user - content: '{{input "note"}}' -output: - format: text - validation_mode: basic +prompts: + example.run: + "1.0.0": + default_profile: offline-example + description: Run a prompt with a deterministic injected model client. + inputs: + - name: note + required: true + content_type: text/plain + messages: + - role: system + content: Summarize the note in one sentence. + - role: user + content: '{{input "note"}}' + output: + format: text + validation_mode: basic diff --git a/generation_error_contract_test.go b/generation_error_contract_test.go index 47907ce..5588f67 100644 --- a/generation_error_contract_test.go +++ b/generation_error_contract_test.go @@ -10,6 +10,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestBuiltInGenerationError(t *testing.T) { @@ -73,7 +74,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) { body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}` return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatalf("NewEngine: %v", err) } @@ -100,7 +101,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, transportCause })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatal(err) } @@ -143,7 +144,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, tc.cause })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatal(err) } @@ -217,7 +218,7 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) Body: io.NopCloser(strings.NewReader(body)), }, nil })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatalf("NewEngine: %v", err) } diff --git a/internal/configuration/prompts.go b/internal/configuration/prompts.go index 9304f89..835e549 100644 --- a/internal/configuration/prompts.go +++ b/internal/configuration/prompts.go @@ -1,6 +1,7 @@ package configuration import ( + "context" "errors" "fmt" "io" @@ -310,3 +311,25 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...) return &definition, inspection, nil } + +// GetPromptSnapshot returns an independently owned captured definition and its +// construction-time hash. Lookup retains literal ID and version keys. +func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string) (*domain.PromptDefinition, string, error) { + if err := ctx.Err(); err != nil { + return nil, "", err + } + definition, inspection, err := state.prompt(id, version) + if errors.Is(err, errPromptNotFound) { + return nil, "", promptdef.ErrPromptDefinitionNotFound + } + if err != nil { + return nil, "", err + } + return definition, inspection.PromptHash, nil +} + +// GetPromptDefinition implements the runner's prompt lookup boundary. +func (state *loaded) GetPromptDefinition(ctx context.Context, id, version string) (*domain.PromptDefinition, error) { + definition, _, err := state.GetPromptSnapshot(ctx, id, version) + return definition, err +} diff --git a/internal/configuration/targets.go b/internal/configuration/targets.go index 8468c46..a0592d5 100644 --- a/internal/configuration/targets.go +++ b/internal/configuration/targets.go @@ -177,3 +177,14 @@ func (state *loaded) GetBackend(id string) (domain.Backend, error) { } return state.registry.GetBackend(id) } + +// CapacityPolicies returns the selected backend admission policies. +func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy { + if state == nil { + return nil + } + return state.registry.CapacityPolicies() +} + +// ExactIDs indicates that ConfigKit definition keys are literal identities. +func (state *loaded) ExactIDs() bool { return true } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4d72f98..82c2a11 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -11,49 +11,42 @@ import ( "strings" "time" - openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2" - rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/artifact" - "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/capacity" + "gitea.maximumdirect.net/eric/promptkit/internal/catalog" + "gitea.maximumdirect.net/eric/promptkit/internal/configuration" "gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/generation" - "gitea.maximumdirect.net/eric/promptkit/internal/legacycatalog" "gitea.maximumdirect.net/eric/promptkit/internal/llm" - "gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/prompt" - "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/usecase" "gitea.maximumdirect.net/eric/promptkit/internal/validate" ) // Config carries construction settings selected by the public facade. type Config struct { - PromptDir string - ProfileDir string + Sources ConfigurationSources SchemaDir string Timeout time.Duration HTTPClient *http.Client TransientRetries *int } +// ConfigurationSources identifies caller-selected ConfigKit layers. +type ConfigurationSources struct { + Underlays []configkit.Underlay + Main *configkit.Input +} + // Options collects construction choices from the public facade. type Options struct { - llmClient llm.Client - artifactReader artifact.Reader - promptDefinitions promptdef.Repository - profiles profile.Repository - fallbackProfiles profile.Repository - memoryProfiles profile.Repository - backends []domain.Backend - validator validate.Validator - promptSource bool - profileSource bool - fallbackProfileSource bool - memorySource bool - validatorSource bool - artifactSource bool + llmClient llm.Client + artifactReader artifact.Reader + validator validate.Validator + validatorSource bool + artifactSource bool } // SetLLMClient selects the client collaborator for one engine. @@ -65,72 +58,6 @@ func (o *Options) SetArtifactReader(reader artifact.Reader) { o.artifactSource = true } -// SetPromptFS selects a prompt source rooted in fsys. -func (o *Options) SetPromptFS(fsys fs.FS, root string) error { - if err := validateSourceFS(fsys, root); err != nil { - return err - } - o.promptDefinitions = promptdef.NewFSRepository(fsys, root) - o.promptSource = true - return nil -} - -// SetPromptFile selects a single prompt file. -func (o *Options) SetPromptFile(path string) error { - fsys, root, directory, err := fileSource(path) - if err != nil { - return err - } - o.promptDefinitions = promptdef.NewFileRepository(fsys, root, directory) - o.promptSource = true - return nil -} - -// SetProfileFS selects an ordinary profile source rooted in fsys. -func (o *Options) SetProfileFS(fsys fs.FS, root string) error { - if err := validateSourceFS(fsys, root); err != nil { - return err - } - o.profiles = profile.NewFSRepository(fsys, root) - o.profileSource = true - return nil -} - -// SetProfileFile selects a single ordinary profile file. -func (o *Options) SetProfileFile(path string) error { - fsys, root, _, err := fileSource(path) - if err != nil { - return err - } - o.profiles = profile.NewFSRepository(fsys, root) - o.profileSource = true - return nil -} - -// SetFallbackProfileFS selects a fallback profile source rooted in fsys. -func (o *Options) SetFallbackProfileFS(fsys fs.FS, root string) error { - if err := validateSourceFS(fsys, root); err != nil { - return err - } - o.fallbackProfiles = profile.NewFSRepository(fsys, root) - o.fallbackProfileSource = true - return nil -} - -// SetMemoryProfiles validates and stores application-owned profile definitions. -func (o *Options) SetMemoryProfiles(definitions []domain.ExecutionProfile) error { - repository, err := profile.NewMemoryRepository(definitions) - if err != nil { - return err - } - o.memoryProfiles = repository - o.memorySource = true - return nil -} - -// AddBackend appends one backend registration. -func (o *Options) AddBackend(backend domain.Backend) { o.backends = append(o.backends, backend) } - // SetSchemaFS selects a schema source rooted in fsys. func (o *Options) SetSchemaFS(fsys fs.FS, root string) error { if err := validateSourceFS(fsys, root); err != nil { @@ -163,33 +90,20 @@ func New(cfg Config, options Options) (*Engine, error) { if err != nil { return nil, err } - promptDefinitions := options.promptDefinitions - if !options.promptSource { - if strings.TrimSpace(cfg.PromptDir) == "" { - return nil, fmt.Errorf("prompt directory is required") - } - promptDefinitions = promptdef.NewFilesystemRepository(cfg.PromptDir) - } - - maintained, err := legacycatalog.Load( - legacycatalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root}, - legacycatalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root}, - ) + underlays, err := catalog.Underlays() if err != nil { - return nil, fmt.Errorf("failed to load maintained catalogs: %w", err) + return nil, fmt.Errorf("select maintained configuration: %w", err) } - - maintainedProfiles, err := newMaintainedProfileRepository(maintained.Profiles, maintained.Backends) + underlays = append(underlays, cfg.Sources.Underlays...) + main := configkit.Input{Source: configkit.NoSource()} + if cfg.Sources.Main != nil { + main = *cfg.Sources.Main + } + snapshot, err := configuration.Load(main, underlays) if err != nil { - return nil, fmt.Errorf("failed to construct Promptkit-maintained profiles: %w", err) + return nil, err } - - profiles := newProfileRepository(cfg.ProfileDir, options, maintainedProfiles) - registry, err := backend.NewRegistry(maintained.Backends, options.backends) - if err != nil { - return nil, fmt.Errorf("failed to construct backend registry: %w", err) - } - manager, err := capacity.NewManager(registry.CapacityPolicies()) + manager, err := capacity.NewManager(snapshot.CapacityPolicies()) if err != nil { return nil, fmt.Errorf("failed to construct backend capacity manager: %w", err) } @@ -217,23 +131,7 @@ func New(cfg Config, options Options) (*Engine, error) { reader = artifact.NewCompositeReader() } - return &Engine{runner: usecase.NewRunnerWithRepairer(promptDefinitions, profiles, registry, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil -} - -func newProfileRepository(profileDir string, options Options, maintained profile.Repository) profile.Repository { - repository := maintained - if options.fallbackProfileSource { - repository = profile.NewOverlayRepository(options.fallbackProfiles, repository) - } - if options.profileSource { - repository = profile.NewOverlayRepository(options.profiles, repository) - } else if strings.TrimSpace(profileDir) != "" { - repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository) - } - if options.memorySource { - repository = profile.NewOverlayRepository(options.memoryProfiles, repository) - } - return profile.NewResolvingRepository(repository) + return &Engine{runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, snapshot, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil } func validateSourceFS(fsys fs.FS, root string) error { diff --git a/internal/usecase/profile_inspection.go b/internal/usecase/profile_inspection.go index 65c457e..ba74515 100644 --- a/internal/usecase/profile_inspection.go +++ b/internal/usecase/profile_inspection.go @@ -19,15 +19,18 @@ func (r *Runner) resolveProfileSelection( ctx context.Context, profileID string, ) (*resolvedProfileSelection, error) { - normalizedID := strings.TrimSpace(profileID) - if normalizedID == "" { + lookupID := profileID + if strings.TrimSpace(lookupID) == "" { return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) } if r == nil || r.profiles == nil { return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad) } + if exact, ok := r.profiles.(interface{ ExactIDs() bool }); !ok || !exact.ExactIDs() { + lookupID = strings.TrimSpace(lookupID) + } - selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID) + selectedProfile, err := r.profiles.GetProfile(ctx, lookupID) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) } @@ -37,7 +40,7 @@ func (r *Runner) resolveProfileSelection( profileValue := *selectedProfile if profileValue.ResolvedTarget != nil { - return &resolvedProfileSelection{id: normalizedID, profile: &profileValue}, nil + return &resolvedProfileSelection{id: lookupID, profile: &profileValue}, nil } profileValue.BackendID = strings.TrimSpace(profileValue.BackendID) @@ -54,7 +57,7 @@ func (r *Runner) resolveProfileSelection( } return &resolvedProfileSelection{ - id: normalizedID, + id: lookupID, profile: &profileValue, backend: selectedBackend, }, nil @@ -80,8 +83,7 @@ func (r *Runner) InspectProfile( ctx context.Context, profileID string, ) (*domain.ProfileInspection, error) { - normalizedID := strings.TrimSpace(profileID) - if normalizedID == "" { + if strings.TrimSpace(profileID) == "" { return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) } select { @@ -90,7 +92,7 @@ func (r *Runner) InspectProfile( default: } - selection, err := r.resolveProfileSelection(ctx, normalizedID) + selection, err := r.resolveProfileSelection(ctx, profileID) if err != nil { return nil, err } diff --git a/internal/usecase/prompt_inspection.go b/internal/usecase/prompt_inspection.go index 31f2e2f..4ee74a9 100644 --- a/internal/usecase/prompt_inspection.go +++ b/internal/usecase/prompt_inspection.go @@ -25,7 +25,16 @@ func (r *Runner) resolvePromptDefinition( return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad) } - definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion) + var definition *domain.PromptDefinition + var hash string + var err error + if indexed, ok := r.promptDefs.(interface { + GetPromptSnapshot(context.Context, string, string) (*domain.PromptDefinition, string, error) + }); ok { + definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID, promptVersion) + } else { + definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion) + } if err != nil { return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) } @@ -33,9 +42,11 @@ func (r *Runner) resolvePromptDefinition( return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad) } - hash, err := hashPromptDefinition(definition) - if err != nil { - return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err) + if hash == "" { + hash, err = hashPromptDefinition(definition) + if err != nil { + return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err) + } } return &resolvedPromptDefinition{definition: definition, hash: hash}, nil } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 41af2cc..e3909cb 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -339,9 +339,9 @@ func (r *Runner) resolvePreparation( return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err) } - selectedProfileID := strings.TrimSpace(req.ProfileID) - if selectedProfileID == "" { - selectedProfileID = strings.TrimSpace(def.DefaultProfile) + selectedProfileID := req.ProfileID + if strings.TrimSpace(selectedProfileID) == "" { + selectedProfileID = def.DefaultProfile } if selectedProfileID == "" { return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired) diff --git a/observation_boundary_contract_test.go b/observation_boundary_contract_test.go index f598d81..6f552d8 100644 --- a/observation_boundary_contract_test.go +++ b/observation_boundary_contract_test.go @@ -8,6 +8,7 @@ import ( "time" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) @@ -23,9 +24,9 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) { ctx, cancel := context.WithTimeout(context.WithValue(context.Background(), contextKey{}, "caller-value"), time.Minute) defer cancel() deadline, _ := ctx.Deadline() - child, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}), + child, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}), promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { gotDeadline, ok := callCtx.Deadline() if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() { diff --git a/observation_contract_test.go b/observation_contract_test.go index de63ab3..abe9fac 100644 --- a/observation_contract_test.go +++ b/observation_contract_test.go @@ -9,13 +9,14 @@ import ( "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}), promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) @@ -128,7 +129,7 @@ func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T) } func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(fstest.MapFS{}, ".")) + engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(fstest.MapFS{}, ".")) if err != nil { t.Fatal(err) } diff --git a/output_contract_contract_test.go b/output_contract_contract_test.go index 9051a75..d7cd7ad 100644 --- a/output_contract_contract_test.go +++ b/output_contract_contract_test.go @@ -6,13 +6,14 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) { - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", diff --git a/prepared_execution_contract_test.go b/prepared_execution_contract_test.go index 87a3300..357170f 100644 --- a/prepared_execution_contract_test.go +++ b/prepared_execution_contract_test.go @@ -13,6 +13,7 @@ import ( "time" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) { @@ -26,10 +27,10 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing. client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: `{"value":3}`}, } - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(promptSource, "."), - promptkit.WithProfileFS(profileSource, "."), + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(promptSource, "."), + testfixture.WithProfileFS(profileSource, "."), promptkit.WithSchemaFS(schemaSource, "."), promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(client), @@ -430,10 +431,10 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: "generated output"}, } - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", @@ -564,10 +565,10 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) { client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: "unexpected"}, } - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", @@ -766,10 +767,10 @@ func newPreparedContractEngine( message string, ) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine( - promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine( + testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prepared", "profile", message), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", diff --git a/prepared_observation_contract_test.go b/prepared_observation_contract_test.go index f006bb3..2c8033c 100644 --- a/prepared_observation_contract_test.go +++ b/prepared_observation_contract_test.go @@ -7,6 +7,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) { @@ -131,9 +132,9 @@ func TestPreparedObservationClaimRaceAndPanicCleanup(t *testing.T) { func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testing.T) { const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY" t.Setenv(envName, "available") - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}), promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})) if err != nil { t.Fatal(err) diff --git a/profile_inheritance_contract_test.go b/profile_inheritance_contract_test.go index 6191953..3b8cc12 100644 --- a/profile_inheritance_contract_test.go +++ b/profile_inheritance_contract_test.go @@ -10,13 +10,14 @@ import ( "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{ + engine, err := testfixture.NewEngine(testfixture.Config{ PromptDir: frameworkPromptDir, SchemaDir: frameworkSchemaDir, - }, promptkit.WithProfiles(promptkit.Profile{ + }, testfixture.WithProfiles(testfixture.Profile{ ID: "weather-light", BaseProfileID: "promptkit-openrouter-standard", ReasoningEffort: "high", @@ -81,9 +82,9 @@ func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { } func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{}, "."), - promptkit.WithProfileFS(fstest.MapFS{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{}, "."), + testfixture.WithProfileFS(fstest.MapFS{ "alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: promptkit-openrouter-standard\n")}, }, "."), ) @@ -100,21 +101,21 @@ func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) { } func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{}, "."), - promptkit.WithBackend(promptkit.Backend{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{}, "."), + testfixture.WithBackend(testfixture.Backend{ ID: "credential-backend", Endpoint: "https://credential.example/v1", APIKeyEnv: "OPTIONAL_BACKEND_KEY", }), - promptkit.WithProfiles( - promptkit.Profile{ + testfixture.WithProfiles( + testfixture.Profile{ ID: "credential-base", BackendID: "credential-backend", Model: "model", APIKeyRequired: true, }, - promptkit.Profile{ID: "credential-child", BaseProfileID: "credential-base"}, + testfixture.Profile{ID: "credential-child", BaseProfileID: "credential-base"}, ), ) if err != nil { @@ -130,17 +131,13 @@ func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) { } func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) { - newEngine := func(t *testing.T, profiles fs.FS) *promptkit.Engine { + newEngine := func(t *testing.T, profiles fs.FS) (*promptkit.Engine, error) { t.Helper() - options := []promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")} + options := []any{testfixture.WithPromptFS(fstest.MapFS{}, ".")} if profiles != nil { - options = append(options, promptkit.WithProfileFS(profiles, ".")) + options = append(options, testfixture.WithProfileFS(profiles, ".")) } - engine, err := promptkit.NewEngine(promptkit.Config{}, options...) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - return engine + return testfixture.NewEngine(testfixture.Config{}, options...) } tests := []struct { @@ -181,7 +178,17 @@ func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - result, err := newEngine(t, tc.profiles).InspectProfile(context.Background(), tc.profile) + engine, err := newEngine(t, tc.profiles) + if tc.profiles != nil { + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("construction = (%#v, %v), want invalid configuration", engine, err) + } + return + } + if err != nil { + t.Fatalf("construct engine: %v", err) + } + result, err := engine.InspectProfile(context.Background(), tc.profile) if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) { t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot) } @@ -200,9 +207,9 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) { "base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")}, }} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."), - promptkit.WithProfileFS(profiles, "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."), + testfixture.WithProfileFS(profiles, "."), promptkit.WithLLMClient(client), ) if err != nil { @@ -220,8 +227,8 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) { t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests) } inspection, err := engine.InspectProfile(context.Background(), "child") - if err != nil || inspection.EffectiveModelParams.Model != "second-model" { - t.Fatalf("fresh inspection=(%+v, %v)", inspection, err) + if err != nil || inspection.EffectiveModelParams.Model != "first-model" { + t.Fatalf("existing engine inspection=(%+v, %v)", inspection, err) } } diff --git a/profiles.go b/profiles.go index b01c826..15f6bd7 100644 --- a/profiles.go +++ b/profiles.go @@ -1,64 +1,71 @@ package promptkit -import ( - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) +import "gitea.maximumdirect.net/eric/ConfigKit" -// OpenAICompatibleProfile returns an in-memory Profile for an OpenAI-compatible -// chat-completions endpoint. A non-blank BaseProfileID permits its target -// fields to be inherited when the profile is selected or inspected. -// -// It does not register global state, maintain a model catalog, or resolve -// credentials. If APIKeyRequired is true, callers satisfy it with -// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv. -// Raw API keys do not belong in profiles. -// -// The function copies the ExtraParams map itself but does not recursively copy -// nested values. Validation and a deep copy occur when NewEngine applies a -// WithProfiles option containing the returned Profile. -func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile { - return Profile{ - ID: cfg.ID, - BaseProfileID: cfg.BaseProfileID, - BackendID: cfg.BackendID, - Endpoint: cfg.Endpoint, - Model: cfg.Model, - Temperature: cfg.Temperature, - MaxTokens: cfg.MaxTokens, - TopP: cfg.TopP, - TimeoutSeconds: cfg.TimeoutSeconds, - ServiceTier: cfg.ServiceTier, - ReasoningEffort: cfg.ReasoningEffort, - APIKeyRequired: cfg.APIKeyRequired, - ExtraParams: copyShallowAnyMap(cfg.ExtraParams), - } +// ProfileConfig is a presence-aware convenience value for a keyed execution +// profile. Nil fields are omitted; non-nil pointers preserve zero and blank +// values. A nil ExtraParams map is omitted; a non-nil empty map supplies an +// empty mapping, not a clear operation. Raw ConfigKit Mapping inputs can +// express explicit null. ProfileConfig has no stable JSON representation. +type ProfileConfig struct { + Extends, Backend, Endpoint, Model *string + Temperature, TopP *float64 + MaxTokens, TimeoutSeconds *int + ServiceTier, ReasoningEffort *string + APIKeyEnv *string + APIKeyRequired *bool + ExtraParams map[string]any } -func copyShallowAnyMap(src map[string]any) map[string]any { - if src == nil { - return nil - } - out := make(map[string]any, len(src)) - for k, v := range src { - out[k] = v - } - return out -} - -func toDomainProfile(publicProfile Profile) domain.ExecutionProfile { - return domain.ExecutionProfile{ - ID: publicProfile.ID, - BaseProfileID: publicProfile.BaseProfileID, - BackendID: publicProfile.BackendID, - Endpoint: publicProfile.Endpoint, - Model: publicProfile.Model, - Temperature: publicProfile.Temperature, - MaxTokens: publicProfile.MaxTokens, - TopP: publicProfile.TopP, - TimeoutSeconds: publicProfile.TimeoutSeconds, - ServiceTier: publicProfile.ServiceTier, - ReasoningEffort: publicProfile.ReasoningEffort, - APIKeyRequired: publicProfile.APIKeyRequired, - ExtraParams: publicProfile.ExtraParams, +// ProfileSource creates a labeled ConfigKit mapping from keyed profile values. +// It encodes fields without validating them; NewEngine validates the final +// effective configuration after all selected source layers are combined. +// Mapping captures a copy of supported input values at creation; unsupported +// values fail when the source is loaded. +func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source { + definitions := make(map[string]any, len(profiles)) + for id, config := range profiles { + fields := make(map[string]any) + if config.Extends != nil { + fields["extends"] = *config.Extends + } + if config.Backend != nil { + fields["backend"] = *config.Backend + } + if config.Endpoint != nil { + fields["endpoint"] = *config.Endpoint + } + if config.Model != nil { + fields["model"] = *config.Model + } + if config.Temperature != nil { + fields["temperature"] = *config.Temperature + } + if config.TopP != nil { + fields["top_p"] = *config.TopP + } + if config.MaxTokens != nil { + fields["max_tokens"] = *config.MaxTokens + } + if config.TimeoutSeconds != nil { + fields["timeout_seconds"] = *config.TimeoutSeconds + } + if config.ServiceTier != nil { + fields["service_tier"] = *config.ServiceTier + } + if config.ReasoningEffort != nil { + fields["reasoning_effort"] = *config.ReasoningEffort + } + if config.APIKeyEnv != nil { + fields["api_key_env"] = *config.APIKeyEnv + } + if config.APIKeyRequired != nil { + fields["api_key_required"] = *config.APIKeyRequired + } + if config.ExtraParams != nil { + fields["extra_params"] = providerMapForMapping(config.ExtraParams) + } + definitions[id] = fields } + return configkit.Mapping(label, map[string]any{"profiles": definitions}) } diff --git a/provider_values.go b/provider_values.go new file mode 100644 index 0000000..6620463 --- /dev/null +++ b/provider_values.go @@ -0,0 +1,121 @@ +package promptkit + +import ( + "encoding/json" + "reflect" + "strconv" +) + +type providerVisit struct { + typ reflect.Type + ptr uintptr +} + +type invalidProviderNumber struct{} + +// providerMapForMapping converts JSON-shaped Go containers to ConfigKit's +// built-in mapping containers. Unsupported values remain for Mapping to report +// as deferred source diagnostics during construction. +func providerMapForMapping(values map[string]any) map[string]any { + if values == nil { + return nil + } + converted := make(map[string]any, len(values)) + active := make(map[providerVisit]bool) + for key, value := range values { + converted[key] = providerValueForMapping(reflect.ValueOf(value), active, 0) + } + return converted +} + +func providerValueForMapping(value reflect.Value, active map[providerVisit]bool, depth int) any { + if !value.IsValid() { + return nil + } + if depth > 100 { + return value.Interface() + } + for value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer { + if value.IsNil() { + return nil + } + if value.Kind() == reflect.Pointer { + visit := providerVisit{typ: value.Type(), ptr: value.Pointer()} + if active[visit] { + return value.Interface() + } + active[visit] = true + defer delete(active, visit) + } + value = value.Elem() + } + if value.CanInterface() { + if number, ok := value.Interface().(json.Number); ok { + if _, err := json.Marshal(number); err != nil { + return invalidProviderNumber{} + } + if integer, err := strconv.ParseInt(string(number), 10, 64); err == nil { + return integer + } + if integer, err := strconv.ParseUint(string(number), 10, 64); err == nil { + return integer + } + if decimal, err := strconv.ParseFloat(string(number), 64); err == nil { + return decimal + } + return number + } + } + switch value.Kind() { + case reflect.Map: + if value.IsNil() { + return nil + } + if value.Type().Key().Kind() != reflect.String { + return value.Interface() + } + visit := providerVisit{typ: value.Type(), ptr: value.Pointer()} + if active[visit] { + return value.Interface() + } + active[visit] = true + defer delete(active, visit) + converted := make(map[string]any, value.Len()) + iter := value.MapRange() + for iter.Next() { + converted[iter.Key().String()] = providerValueForMapping(iter.Value(), active, depth+1) + } + return converted + case reflect.Slice, reflect.Array: + if value.Kind() == reflect.Slice { + if value.IsNil() { + return nil + } + visit := providerVisit{typ: value.Type(), ptr: value.Pointer()} + if visit.ptr != 0 { + if active[visit] { + return value.Interface() + } + active[visit] = true + defer delete(active, visit) + } + } + converted := make([]any, value.Len()) + for index := range converted { + converted[index] = providerValueForMapping(value.Index(index), active, depth+1) + } + return converted + case reflect.String: + return value.String() + case reflect.Bool: + return value.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return value.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return value.Uint() + case reflect.Float32, reflect.Float64: + return value.Float() + default: + return value.Interface() + } +} diff --git a/public_contract_test.go b/public_contract_test.go index ded0a7c..cdda81a 100644 --- a/public_contract_test.go +++ b/public_contract_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io/fs" "reflect" "strings" "sync" @@ -14,33 +13,25 @@ import ( "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) -type inspectionCountingFS struct { - opens atomic.Int64 -} - -func (f *inspectionCountingFS) Open(string) (fs.File, error) { - f.opens.Add(1) - return nil, fs.ErrNotExist -} - func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) { const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY" t.Setenv(environmentName, "") client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{}, "."), - promptkit.WithProfileFS(fstest.MapFS{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{}, "."), + testfixture.WithProfileFS(fstest.MapFS{ "environment.yaml": &fstest.MapFile{Data: []byte(`id: environment endpoint: http://environment.example/v1 model: environment-model api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY `)}, }, "."), - promptkit.WithProfiles( - promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true}, - promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"}, + testfixture.WithProfiles( + testfixture.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true}, + testfixture.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"}, ), promptkit.WithLLMClient(client), ) @@ -54,7 +45,7 @@ api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY wantDirectKey bool wantEndpoint string }{ - {profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"}, + {profileID: "environment", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"}, {profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"}, {profileID: "none", wantEndpoint: "http://none.example/v1"}, } { @@ -63,7 +54,7 @@ api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY if err != nil { t.Fatalf("inspect profile: %v", err) } - if inspection.ProfileID != strings.TrimSpace(tc.profileID) || + if inspection.ProfileID != tc.profileID || inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint || inspection.EffectiveModelParams.BackendID != "" || inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv || @@ -77,10 +68,10 @@ api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY } } -func TestFileProfileNormalizedIDMatchesInspectionAndPreparation(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "normalized-profile", "message"), "."), - promptkit.WithProfileFS(fstest.MapFS{ +func TestFileProfileExactIDMatchesInspectionAndPreparation(t *testing.T) { + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", " normalized-profile ", "message"), "."), + testfixture.WithProfileFS(fstest.MapFS{ "profile.yaml": &fstest.MapFile{Data: []byte(` id: " normalized-profile " endpoint: http://profile.example/v1 @@ -92,19 +83,19 @@ model: normalized-model t.Fatalf("construct engine: %v", err) } - inspection, err := engine.InspectProfile(context.Background(), " normalized-profile ") + inspection, err := engine.InspectProfile(context.Background(), " normalized-profile ") if err != nil { t.Fatalf("inspect normalized profile: %v", err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "prompt", - ProfileID: " normalized-profile ", + ProfileID: " normalized-profile ", }) if err != nil { t.Fatalf("prepare with normalized profile: %v", err) } - if inspection.ProfileID != "normalized-profile" || + if inspection.ProfileID != " normalized-profile " || prepared.SelectedProfileID != inspection.ProfileID || inspection.EffectiveModelParams.Model != "normalized-model" || prepared.EffectiveModelParams.Model != inspection.EffectiveModelParams.Model { @@ -113,11 +104,11 @@ model: normalized-model } func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) { - newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine { + newEngine := func(t *testing.T, options ...any) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine( - promptkit.Config{}, - append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)..., + engine, err := testfixture.NewEngine( + testfixture.Config{}, + append([]any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}, options...)..., ) if err != nil { t.Fatalf("construct inspection engine: %v", err) @@ -131,7 +122,7 @@ func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) { t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err) } - valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{ + valid := newEngine(t, testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", })) if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil || @@ -143,29 +134,25 @@ func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) { t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err) } - malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{ + malformed, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfileFS(fstest.MapFS{ "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nextra_params:\n invalid: .nan\n")}, }, ".")) - if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil || - !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err) + if malformed != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("malformed profile construction=(%#v, %v), want ErrInvalidConfig", malformed, err) } - unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{ + unknownBackend, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfiles(testfixture.Profile{ ID: "unknown-backend", BackendID: "unknown", Model: "model", })) - if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil || - !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err) + if unknownBackend != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("unknown backend construction=(%#v, %v), want ErrInvalidConfig", unknownBackend, err) } - countingFS := &inspectionCountingFS{} - canceled := newEngine(t, promptkit.WithProfileFS(countingFS, ".")) + canceled := newEngine(t, testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://profile.example/v1"})) ctx, cancel := context.WithCancel(context.Background()) cancel() - if result, err := canceled.InspectProfile(ctx, "profile"); result != nil || - !errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 { - t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load()) + if result, err := canceled.InspectProfile(ctx, "profile"); result != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("canceled inspection result=(%#v, %v)", result, err) } } @@ -173,9 +160,9 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) extraParams := map[string]any{ "nested": map[string]any{"value": "original"}, } - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams, }), ) @@ -210,8 +197,8 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{ "report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report version: "1.0.0" messages: @@ -278,7 +265,7 @@ output: func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) { newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, ".")) + engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, ".")) if err != nil { t.Fatalf("construct prompt-inspection engine: %v", err) } @@ -358,29 +345,25 @@ output: }, } { t.Run(name, func(t *testing.T) { - if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil || - !errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err) + engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, ".")) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("broken prompt construction=(%#v, %v), want ErrInvalidConfig", engine, err) } }) } - countingFS := &inspectionCountingFS{} - canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, ".")) - if err != nil { - t.Fatalf("construct canceled prompt-inspection engine: %v", err) - } + canceled := valid ctx, cancel := context.WithCancel(context.Background()) cancel() if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil || - !errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 { - t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load()) + !errors.Is(err, context.Canceled) { + t.Fatalf("canceled inspection result=(%#v, %v)", result, err) } } func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{ "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt version: "1" default_profile: profile @@ -396,7 +379,7 @@ output: validation_mode: basic `)}, }, "."), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", }), ) @@ -482,9 +465,9 @@ func TestBackendIdentityJSONNamesAndOmission(t *testing.T) { func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", Endpoint: "http://example.test/v1", Model: "model", }), promptkit.WithLLMClient(client), @@ -518,21 +501,13 @@ func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) { } } -func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}), +func TestUnknownProfileBackendFailsConstruction(t *testing.T) { + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "unknown", Model: "model"}), ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - if errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("unknown backend should not have invalid-request identity: %v", err) + if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("construction=(%#v, %v), want ErrInvalidConfig", engine, err) } } @@ -546,8 +521,8 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) { } endpoint := "http://local.example/v1" - backend := promptkit.LocalBackend(endpoint, limit) - want := promptkit.Backend{ + backend := testfixture.LocalBackend(endpoint, limit) + want := testfixture.Backend{ ID: localBackendID, Endpoint: endpoint, ConcurrencyLimit: limit, @@ -556,10 +531,10 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) { t.Fatalf("LocalBackend()=%+v, want %+v", backend, want) } - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."), - promptkit.WithBackend(backend), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."), + testfixture.WithBackend(backend), + testfixture.WithProfiles(testfixture.Profile{ ID: "local-profile", BackendID: localBackendID, Model: "model", @@ -582,20 +557,20 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) { func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) { t.Setenv("CUSTOM_LLM_KEY", "test-key") client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."), - promptkit.WithBackend(promptkit.Backend{ - ID: " custom ", + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."), + testfixture.WithBackend(testfixture.Backend{ + ID: "custom", Endpoint: " http://backend.example/v1 ", APIKeyEnv: " CUSTOM_LLM_KEY ", ExtraParams: map[string]any{ "provider": "custom", }, }), - promptkit.WithProfiles( - promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"}, - promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"}, - promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"}, + testfixture.WithProfiles( + testfixture.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"}, + testfixture.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"}, + testfixture.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Model: "profile-model"}, ), promptkit.WithLLMClient(client), ) @@ -655,16 +630,15 @@ func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing. } func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."), - promptkit.WithProfileFS(fstest.MapFS{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."), + testfixture.WithProfileFS(fstest.MapFS{ "profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile backend: file-backend -endpoint: " " model: file-model `)}, }, "."), - promptkit.WithBackend(promptkit.Backend{ + testfixture.WithBackend(testfixture.Backend{ ID: "file-backend", Endpoint: "http://file-backend.example/v1", }), @@ -689,13 +663,13 @@ model: file-model } func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."), - promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}), - promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}), - promptkit.WithProfiles( - promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"}, - promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"}, + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."), + testfixture.WithBackend(testfixture.Backend{ID: "first", Endpoint: "http://first.example/v1"}), + testfixture.WithBackend(testfixture.Backend{ID: "second", Endpoint: "http://second.example/v1"}), + testfixture.WithProfiles( + testfixture.Profile{ID: "first-profile", BackendID: "first", Model: "model"}, + testfixture.Profile{ID: "second-profile", BackendID: "second", Model: "model"}, ), ) if err != nil { @@ -718,10 +692,10 @@ func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) { newEngine := func(endpoint string) *promptkit.Engine { t.Helper() - value, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}), + value, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithBackend(testfixture.Backend{ID: "same-id", Endpoint: endpoint}), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "same-id", Model: "model"}), ) if err != nil { t.Fatalf("construct isolated engine: %v", err) @@ -746,7 +720,7 @@ func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) { func TestWithBackendCopiesQueueCapacity(t *testing.T) { queueCapacity := 4 - option := promptkit.WithBackend(promptkit.Backend{ + option := testfixture.WithBackend(testfixture.Backend{ ID: "custom", Endpoint: "http://custom.example/v1", ConcurrencyLimit: 1, @@ -754,10 +728,10 @@ func TestWithBackendCopiesQueueCapacity(t *testing.T) { }) queueCapacity = -1 - _, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + _, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), option, - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", BackendID: "custom", Model: "model", }), ) @@ -766,41 +740,31 @@ func TestWithBackendCopiesQueueCapacity(t *testing.T) { } } -func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) { +func TestBackendConfigRejectsInvalidDefinitions(t *testing.T) { cycle := map[string]any{} cycle["self"] = cycle tests := []struct { name string - backends []promptkit.Backend + backends []testfixture.Backend }{ - {name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}}, - {name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}}, - {name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}}, - {name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}}, - {name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}}, - {name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}}, - {name: "excessively deep extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}}, - {name: "duplicate consumer id", backends: []promptkit.Backend{ - {ID: " custom ", Endpoint: "http://one.example/v1"}, - {ID: "custom", Endpoint: "http://two.example/v1"}, - }}, - {name: "reserved OpenRouter ID", backends: []promptkit.Backend{{ - ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1", - }}}, - {name: "reserved Rakestrawhome ID", backends: []promptkit.Backend{{ - ID: promptkit.BackendRakestrawHome, Endpoint: "http://replacement.example/v1", - }}}, + {name: "blank id", backends: []testfixture.Backend{{Endpoint: "http://example.test/v1"}}}, + {name: "invalid endpoint", backends: []testfixture.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}}, + {name: "invalid environment", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}}, + {name: "reserved extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}}, + {name: "cyclic extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}}, + {name: "malformed JSON number", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}}, + {name: "excessively deep extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - options := []promptkit.Option{ - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + options := []any{ + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), } for _, backend := range tt.backends { - options = append(options, promptkit.WithBackend(backend)) + options = append(options, testfixture.WithBackend(backend)) } - _, err := promptkit.NewEngine(promptkit.Config{}, options...) + _, err := testfixture.NewEngine(testfixture.Config{}, options...) if !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } @@ -811,12 +775,12 @@ func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) { nested := map[string]any{"value": "original"} extraParams := map[string]any{"nested": nested} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithBackend(promptkit.Backend{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithBackend(testfixture.Backend{ ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams, }), - promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}), + testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "custom", Model: "model"}), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -956,13 +920,13 @@ func TestEngineRunReturnsFinalResultAfterRepairExhaustion(t *testing.T) { } func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { - profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"} + profile := testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"} t.Run("prompt source", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("first", "profile", "first"), "."), - promptkit.WithPromptFS(contractPromptFS("second", "profile", "second"), "."), - promptkit.WithProfiles(profile), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("first", "profile", "first"), "."), + testfixture.WithPromptFS(contractPromptFS("second", "profile", "second"), "."), + testfixture.WithProfiles(profile), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -977,10 +941,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { }) t.Run("profile source", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfileFS(contractProfileFS("profile", "first-model"), "."), - promptkit.WithProfileFS(contractProfileFS("profile", "second-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfileFS(contractProfileFS("profile", "first-model"), "."), + testfixture.WithProfileFS(contractProfileFS("profile", "second-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -995,10 +959,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { }) t.Run("fallback profile source", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1017,10 +981,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { first.Model = "first-model" second := profile second.Model = "second-model" - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(first), - promptkit.WithProfiles(second), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(first), + testfixture.WithProfiles(second), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1035,9 +999,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { }) t.Run("schema source", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractSchemaPromptFS(), "."), - promptkit.WithProfiles(profile), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractSchemaPromptFS(), "."), + testfixture.WithProfiles(profile), promptkit.WithSchemaFS(contractSchemaFS("first"), "."), promptkit.WithSchemaFS(contractSchemaFS("second"), "."), ) @@ -1056,9 +1020,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { t.Run("model client", func(t *testing.T) { var firstCalls, secondCalls atomic.Int64 - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(profile), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithProfiles(profile), promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}), promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}), ) @@ -1074,9 +1038,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { }) t.Run("artifact reader", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractInputPromptFS(), "."), - promptkit.WithProfiles(profile), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractInputPromptFS(), "."), + testfixture.WithProfiles(profile), promptkit.WithArtifactReader(fixedArtifactReader("first")), promptkit.WithArtifactReader(fixedArtifactReader("second")), ) @@ -1109,11 +1073,11 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { } t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}), - promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}), + testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1124,10 +1088,10 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { }) t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1140,9 +1104,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { t.Run("ordinary option replaces configured directory", func(t *testing.T) { profileDir := t.TempDir() writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model") - engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithProfileFS(contractProfileFS(profileID, "option-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithProfileFS(contractProfileFS(profileID, "option-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1155,9 +1119,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { t.Run("configured directory overrides fallback profile", func(t *testing.T) { profileDir := t.TempDir() writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model") - engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1168,9 +1132,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { }) t.Run("fallback profile overrides built-in profile", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1182,17 +1146,17 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) { t.Run("missing fallback profile uses built-in profile", func(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "test-key") - baseline, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), + baseline, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), ) if err != nil { t.Fatalf("construct baseline engine: %v", err) } want := prepareModel(t, baseline, "prompt") - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1210,8 +1174,8 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { ) t.Run("inspection and preparation resolve both names", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", standardID, "message"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", standardID, "message"), "."), ) if err != nil { t.Fatalf("construct engine: %v", err) @@ -1241,9 +1205,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { }) t.Run("consumer definitions override an owned profile and its base", func(t *testing.T) { - auto, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("auto", autoID, "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + auto, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("auto", autoID, "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: autoID, Endpoint: "http://consumer.example/v1", Model: "consumer-auto-model", @@ -1260,9 +1224,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { t.Fatalf("owned profile override model = %q", autoPrepared.EffectiveModelParams.Model) } - standard, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("standard", standardID, "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + standard, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("standard", standardID, "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "gemini-flash-latest", Endpoint: "http://consumer.example/v1", Model: "consumer-base-model", @@ -1281,9 +1245,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { }) t.Run("prompt default selection remains explicit", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("default", "application-default", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("default", "application-default", "message"), "."), + testfixture.WithProfiles(testfixture.Profile{ ID: "application-default", Endpoint: "http://consumer.example/v1", Model: "consumer-default-model", @@ -1302,69 +1266,56 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { }) } -func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) { +func TestInvalidFallbackProfileSourcesFailConstruction(t *testing.T) { const profileID = "application-profile" - t.Run("construction defers malformed fallback profiles", func(t *testing.T) { - _, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(fstest.MapFS{}, "."), - promptkit.WithFallbackProfileFS(fstest.MapFS{ + t.Run("malformed fallback profile", func(t *testing.T) { + _, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(fstest.MapFS{}, "."), + testfixture.WithFallbackProfileFS(fstest.MapFS{ "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")}, }, "."), ) - if err != nil { - t.Fatalf("construct engine with malformed fallback profile: %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid configuration, got %v", err) } }) - t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithFallbackProfileFS(fstest.MapFS{ + t.Run("unrelated malformed fallback profile blocks construction", func(t *testing.T) { + _, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithFallbackProfileFS(fstest.MapFS{ "broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")}, "valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")}, }, "."), ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare from valid fallback profile: %v", err) - } - if prepared.EffectiveModelParams.Model != "fallback-model" { - t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid configuration, got %v", err) } }) - t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - promptkit.WithFallbackProfileFS(fstest.MapFS{ + t.Run("matching malformed fallback profile", func(t *testing.T) { + _, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), + testfixture.WithFallbackProfileFS(fstest.MapFS{ "mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")}, }, "."), ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid configuration, got %v", err) } }) - t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithProfileFS(fstest.MapFS{ + t.Run("matching malformed ordinary profile", func(t *testing.T) { + _, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithProfileFS(fstest.MapFS{ "application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")}, }, "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) + if !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("expected invalid configuration, got %v", err) } }) } @@ -1372,9 +1323,9 @@ func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) { func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) { const profileID = "application-profile" client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), + testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), promptkit.WithLLMClient(client), ) if err != nil { @@ -1417,13 +1368,13 @@ func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) { } func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithBackend(promptkit.Backend{ + engine, err := testfixture.NewEngine(testfixture.Config{}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + testfixture.WithBackend(testfixture.Backend{ ID: "concurrent", Endpoint: "http://example.test/v1", }), - promptkit.WithProfiles(promptkit.Profile{ + testfixture.WithProfiles(testfixture.Profile{ ID: "profile", BackendID: "concurrent", Model: "model", diff --git a/reliable_execution_contract_test.go b/reliable_execution_contract_test.go index 3615519..0c82df1 100644 --- a/reliable_execution_contract_test.go +++ b/reliable_execution_contract_test.go @@ -13,6 +13,7 @@ import ( "time" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func observedHTTPResponse(status int, body string) *http.Response { @@ -39,7 +40,7 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) { calls++ return r, nil })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatal(err) } @@ -124,10 +125,10 @@ func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) { } }) one := 1 - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), - promptkit.WithBackend(promptkit.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}), - promptkit.WithProfiles(promptkit.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, promptkit.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"})) + engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), + testfixture.WithBackend(testfixture.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}), + testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"})) if err != nil { t.Fatal(err) } @@ -186,11 +187,11 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) { return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil }) zero := 0 - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero}, - promptkit.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), - promptkit.WithBackend(promptkit.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), - promptkit.WithBackend(promptkit.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), - promptkit.WithProfiles(promptkit.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, promptkit.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"})) + engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero}, + testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), + testfixture.WithBackend(testfixture.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), + testfixture.WithBackend(testfixture.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), + testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"})) if err != nil { t.Fatal(err) } @@ -251,8 +252,8 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) { return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil }) one := 1 - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, - promptkit.WithPromptFS(source, "."), promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"})) + engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, + testfixture.WithPromptFS(source, "."), testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"})) if err != nil { t.Fatal(err) } diff --git a/testfixture/engine.go b/testfixture/engine.go new file mode 100644 index 0000000..501e9c8 --- /dev/null +++ b/testfixture/engine.go @@ -0,0 +1,359 @@ +// Package testfixture keeps existing runtime regression scenarios executable +// while their source setup is expressed through the new configuration boundary. +package testfixture + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" + "gopkg.in/yaml.v3" +) + +// Config carries the existing regression fixtures' construction settings. +type Config struct { + PromptDir string + ProfileDir string + SchemaDir string + Timeout time.Duration + HTTPClient *http.Client + TransientRetries *int +} + +// Profile describes an existing regression fixture's execution settings. +type Profile struct { + ID, BaseProfileID, BackendID, Endpoint, Model string + Temperature, TopP float64 + MaxTokens, TimeoutSeconds int + ServiceTier, ReasoningEffort string + APIKeyRequired bool + ExtraParams map[string]any +} + +// OpenAICompatibleProfileConfig describes a profile fixture. +type OpenAICompatibleProfileConfig = Profile + +// OpenAICompatibleProfile produces a profile fixture. +func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile { + profile := Profile(config) + if config.ExtraParams != nil { + profile.ExtraParams = make(map[string]any, len(config.ExtraParams)) + for key, value := range config.ExtraParams { + profile.ExtraParams[key] = value + } + } + return profile +} + +// Backend describes an existing regression fixture's backend settings. +type Backend struct { + ID, Endpoint, APIKeyEnv string + ConcurrencyLimit int + QueueCapacity *int + ExtraParams map[string]any +} + +// LocalBackend returns a conventional local backend fixture. +func LocalBackend(endpoint string, concurrencyLimit int) Backend { + return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit} +} + +type option func(*choices) error + +type choices struct { + promptFS, profileFS, fallbackFS fs.FS + promptRoot, profileRoot, fallbackRoot string + promptFile, profileFile string + profiles []Profile + profilesSet bool + backends []Backend +} + +// WithPromptFS selects prompt fixture files. +func WithPromptFS(fsys fs.FS, root string) option { + return func(c *choices) error { + if err := validSourceFS(fsys, root); err != nil { + return err + } + c.promptFS, c.promptRoot, c.promptFile = fsys, root, "" + return nil + } +} + +// WithPromptFile selects one prompt fixture file. +func WithPromptFile(name string) option { + return func(c *choices) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("prompt file is required") + } + c.promptFS, c.promptFile = nil, name + return nil + } +} + +// WithProfileFS selects profile fixture files. +func WithProfileFS(fsys fs.FS, root string) option { + return func(c *choices) error { + if err := validSourceFS(fsys, root); err != nil { + return err + } + c.profileFS, c.profileRoot, c.profileFile = fsys, root, "" + return nil + } +} + +// WithProfileFile selects one profile fixture file. +func WithProfileFile(name string) option { + return func(c *choices) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("profile file is required") + } + c.profileFS, c.profileFile = nil, name + return nil + } +} + +// WithFallbackProfileFS selects lower-precedence profile fixture files. +func WithFallbackProfileFS(fsys fs.FS, root string) option { + return func(c *choices) error { + if err := validSourceFS(fsys, root); err != nil { + return err + } + c.fallbackFS, c.fallbackRoot = fsys, root + return nil + } +} + +func validSourceFS(filesystem fs.FS, root string) error { + if filesystem == nil || strings.TrimSpace(root) == "" { + return fmt.Errorf("source filesystem and root are required") + } + return nil +} + +// WithProfiles selects in-memory profile fixtures. +func WithProfiles(profiles ...Profile) option { + return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil } +} + +// WithBackend adds one backend fixture. +func WithBackend(backend Backend) option { + if backend.QueueCapacity != nil { + capacity := *backend.QueueCapacity + backend.QueueCapacity = &capacity + } + return func(c *choices) error { c.backends = append(c.backends, backend); return nil } +} + +// NewEngine expresses existing regression scenarios through ConfigKit sources. +func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) { + var choices choices + var runtime []promptkit.Option + for _, candidate := range opts { + switch value := candidate.(type) { + case nil: + case option: + if err := value(&choices); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + case promptkit.Option: + runtime = append(runtime, value) + default: + return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate) + } + } + var underlays []configkit.Underlay + add := func(inputs []configkit.Input, err error) error { + if err != nil { + return err + } + if len(inputs) > 0 { + underlays = append(underlays, configkit.Underlay{Inputs: inputs}) + } + return nil + } + if choices.fallbackFS != nil { + if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } + if choices.profileFile != "" { + if err := add(singleInput(choices.profileFile, "profiles")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } else if choices.profileFS != nil { + if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } else if config.ProfileDir != "" { + if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } + if choices.profilesSet { + profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles)) + for _, value := range choices.profiles { + if _, exists := profiles[value.ID]; exists { + return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID) + } + profiles[value.ID] = profileConfig(value) + } + underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}}) + } + for index, value := range choices.backends { + config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv, + ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams} + underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}}) + } + if choices.promptFile != "" { + if err := add(singleInput(choices.promptFile, "prompts")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } else if choices.promptFS != nil { + if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } else if config.PromptDir != "" { + if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil { + return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) + } + } + return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}, + SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient, + TransientRetries: config.TransientRetries}, runtime...) +} + +func profileConfig(value Profile) promptkit.ProfileConfig { + config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams} + if value.BaseProfileID != "" { + config.Extends = &value.BaseProfileID + } + if value.BackendID != "" { + config.Backend = &value.BackendID + } + if value.Endpoint != "" { + config.Endpoint = &value.Endpoint + } + if value.Model != "" { + config.Model = &value.Model + } + if value.Temperature != 0 { + config.Temperature = &value.Temperature + } + if value.TopP != 0 { + config.TopP = &value.TopP + } + if value.MaxTokens != 0 { + config.MaxTokens = &value.MaxTokens + } + if value.TimeoutSeconds != 0 { + config.TimeoutSeconds = &value.TimeoutSeconds + } + if value.ServiceTier != "" { + config.ServiceTier = &value.ServiceTier + } + if value.ReasoningEffort != "" { + config.ReasoningEffort = &value.ReasoningEffort + } + if value.APIKeyRequired { + config.APIKeyRequired = &value.APIKeyRequired + } + return config +} + +func directoryInputs(directory, collection string) ([]configkit.Input, error) { + if _, err := os.Stat(directory); err != nil { + return nil, err + } + return sourceInputs(directory, os.DirFS(directory), ".", collection) +} + +func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) { + return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root) +} + +func singleInput(name, collection string) ([]configkit.Input, error) { + info, err := os.Stat(name) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("source %s is a directory", name) + } + filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection} + namespace := configkit.NewFilesystem(name, filesystem) + return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil +} + +type convertingFS struct { + fs.FS + collection string +} + +func (source convertingFS) Open(name string) (fs.File, error) { + file, err := source.FS.Open(name) + if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) { + return file, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + data, err := io.ReadAll(file) + closeErr := file.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + converted := convertDocument(data, source.collection) + return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil +} + +type memoryFile struct { + *bytes.Reader + info fs.FileInfo +} + +func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil } +func (file *memoryFile) Close() error { return nil } + +func convertDocument(data []byte, collection string) []byte { + var document map[string]any + if err := yaml.Unmarshal(data, &document); err != nil || document == nil { + return data + } + if _, keyed := document[collection]; keyed { + return data + } + id, idOK := document["id"] + if !idOK { + return data + } + delete(document, "id") + if collection == "profiles" { + if base, exists := document["base_profile"]; exists { + delete(document, "base_profile") + document["extends"] = base + } + data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}}) + return data + } + version, versionOK := document["version"] + if !versionOK { + return data + } + delete(document, "version") + data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}}) + return data +} diff --git a/transient_retries_contract_test.go b/transient_retries_contract_test.go index 6c691d6..490e62d 100644 --- a/transient_retries_contract_test.go +++ b/transient_retries_contract_test.go @@ -11,6 +11,7 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func retryHTTPResponse(status int, content string) *http.Response { @@ -30,7 +31,7 @@ func retryTestEngine(t *testing.T, budget *int, next func(int) *http.Response) ( calls++ return next(calls), nil })} - engine, err := promptkit.NewEngine(config) + engine, err := testfixture.NewEngine(config) if err != nil { t.Fatal(err) } @@ -71,7 +72,7 @@ func TestRetryValidationAndPreparedFreezing(t *testing.T) { negative := -1 config := contractConfig(frameworkSchemaDir) config.TransientRetries = &negative - if _, err := promptkit.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) { + if _, err := testfixture.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("negative engine budget = %v", err) } diff --git a/types.go b/types.go index e1a2016..e6f2cf2 100644 --- a/types.go +++ b/types.go @@ -468,111 +468,6 @@ type ExecutionTargetOverride struct { ExtraParams map[string]any } -// Profile is an in-memory execution profile for library consumers. -// -// A standalone Profile is equivalent to a loaded profile file after local -// validation. A derived profile names BaseProfileID and can inherit target -// fields when selected or inspected. Raw API keys do not belong in profiles; -// use APIKeyRequired to require callers to provide a RunRequest.APIKey or -// explicit request ExecutionTargetOverride.APIKeyEnv, or use profile YAML -// api_key_env with file and FS profile sources. Profile has no stable JSON -// representation. -// -// WithProfiles locally validates and copies Profile values during NewEngine. -// It checks base-reference existence and resolved target completeness when a -// derived profile is selected or inspected. Zero Temperature, MaxTokens, and -// TopP values and blank ServiceTier and ReasoningEffort values leave those -// provider controls unspecified. A zero TimeoutSeconds retains the framework -// deadline, while an empty ExtraParams map inherits backend request defaults. -// Use ExecutionTargetOverride pointer fields to request an explicit numeric -// zero. -type Profile struct { - // ID is the required non-blank profile identifier. WithProfiles trims it. - ID string - // BaseProfileID optionally names one base profile. WithProfiles trims it. A - // non-blank value permits required target fields to be inherited when the - // profile is selected or inspected, which is also when reference existence - // and resolved completeness are checked. A blank value leaves this as a - // standalone profile. - BaseProfileID string - // BackendID optionally selects an engine backend. WithProfiles trims it. - // Backend membership is checked when a request selects the profile; an - // unknown ID makes preparation fail with ErrProfileLoad. - BackendID string - // Endpoint is the model-provider base URL. A standalone Profile requires an - // endpoint when BackendID is blank; a derived Profile may inherit either - // field. A non-blank endpoint overrides the backend endpoint when - // non-blank. WithProfiles trims it and requires an absolute HTTP or HTTPS URL - // with a host and no user information, query, or fragment. - Endpoint string - // Model is the provider model identifier. It is required for a standalone - // Profile and may be inherited by a derived Profile. - Model string - // Temperature is from 0 through 2. Zero leaves the provider control - // unspecified. - Temperature float64 - // MaxTokens is non-negative. Zero leaves the provider control unspecified. - MaxTokens int - // TopP is from 0 through 1. Zero leaves the provider control unspecified - // rather than selecting an explicit zero. - TopP float64 - // TimeoutSeconds is non-negative. Zero retains the framework deadline. - TimeoutSeconds int - // ServiceTier is optional; a blank value leaves it unspecified. - ServiceTier string - // ReasoningEffort is optional; a blank value leaves it unspecified. - ReasoningEffort string - // APIKeyRequired clears a backend's inherited API-key environment name and - // requires a non-blank RunRequest.APIKey unless the request explicitly - // supplies ExecutionTargetOverride.APIKeyEnv. When false, a named - // environment source remains optional. It does not store a credential. - APIKeyRequired bool - // ExtraParams contains provider-specific JSON-compatible values. An empty - // map inherits backend request defaults, when any. WithProfiles validates - // and deeply copies it during NewEngine. Excessively deep or large values - // are rejected for safety. - ExtraParams map[string]any -} - -// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory -// profile. -// -// It contains ordinary profile fields for OpenAI-compatible chat-completions -// endpoints. BaseProfileID and APIKeyRequired follow Profile. Raw API keys do -// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON -// representation and is not validated until its resulting Profile is supplied -// through WithProfiles to NewEngine. -type OpenAICompatibleProfileConfig struct { - // ID becomes Profile.ID. - ID string - // BaseProfileID becomes Profile.BaseProfileID. A non-blank value permits the - // resulting Profile to inherit target fields when it is selected or inspected. - BaseProfileID string - // BackendID becomes Profile.BackendID. - BackendID string - // Endpoint becomes Profile.Endpoint. - Endpoint string - // Model becomes Profile.Model. - Model string - // APIKeyRequired becomes Profile.APIKeyRequired. - APIKeyRequired bool - // Temperature becomes Profile.Temperature. - Temperature float64 - // MaxTokens becomes Profile.MaxTokens. - MaxTokens int - // TopP becomes Profile.TopP. - TopP float64 - // TimeoutSeconds becomes Profile.TimeoutSeconds. - TimeoutSeconds int - // ServiceTier becomes Profile.ServiceTier. - ServiceTier string - // ReasoningEffort becomes Profile.ReasoningEffort. - ReasoningEffort string - // ExtraParams becomes a shallow-copied Profile.ExtraParams map. NewEngine - // performs validation and a deep copy when WithProfiles applies the result. - ExtraParams map[string]any -} - // ExecutionTargetPresence reports which effective numeric settings should be // sent even when zero. It has a stable JSON representation and lets injected // clients preserve omission semantics.