From 242eace4a7fd297d12600b90f0e895253ba83e18 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 19:52:00 +0000 Subject: [PATCH] Expose profile inspection through the engine --- convert.go | 11 +++ doc.go | 38 +++++----- engine.go | 54 ++++++++++++-- public_contract_test.go | 158 ++++++++++++++++++++++++++++++++++++++++ types.go | 19 +++++ 5 files changed, 255 insertions(+), 25 deletions(-) diff --git a/convert.go b/convert.go index 646ac1b..daf05dc 100644 --- a/convert.go +++ b/convert.go @@ -170,6 +170,17 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { } } +func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection { + if inspection == nil { + return nil + } + return &ProfileInspection{ + ProfileID: inspection.ProfileID, + EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams), + APIKeyRequired: inspection.APIKeyRequired, + } +} + func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence { return ExecutionTargetPresence{ Temperature: presence.Temperature, diff --git a/doc.go b/doc.go index 68a2775..c5e88c2 100644 --- a/doc.go +++ b/doc.go @@ -3,25 +3,27 @@ // // Applications construct an [Engine] with [NewEngine], select filesystem or // in-memory sources and optional engine-scoped [Backend] registrations, and -// call [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], or -// [Engine.RunPrepared]. Concrete registries, repositories, validators, and the -// built-in OpenAI-compatible client remain internal implementation details. +// call [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution], +// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories, +// validators, and the built-in OpenAI-compatible client remain internal +// implementation details. // // # Concurrency and ownership // -// An Engine supports concurrent Prepare, PrepareExecution, Run, and RunPrepared -// calls. Engine-local backend policies bound admitted Run and RunPrepared calls -// and model generations where configured, while different backend pools and -// unlimited backends continue independently. An injected [LLMClient] or -// [ArtifactReader] can therefore still receive concurrent calls and must be -// safe for that use. +// An Engine supports concurrent InspectProfile, Prepare, PrepareExecution, Run, +// and RunPrepared calls. Engine-local backend policies bound admitted Run and +// RunPrepared calls and model generations where configured, while different +// backend pools and unlimited backends continue independently. An injected +// [LLMClient] or [ArtifactReader] can therefore still receive concurrent calls +// and must be safe for that use. // // NewEngine copies in-memory profiles and backend definitions. Prepare, // PrepareExecution, and Run copy request maps, slices, pointer values, and -// JSON-compatible extra parameters before using them. Returned values and -// values passed to extension interfaces are likewise isolated from engine -// state. Callers own those copies and may mutate them after the call that -// supplied or returned them. +// JSON-compatible extra parameters before using them. InspectProfile returns +// copied profile inspection values. Returned values and values passed to +// extension interfaces are likewise isolated from engine state. Callers own +// those copies and may mutate them after the call that supplied or returned +// them. // // # Security and sensitive data // @@ -47,11 +49,11 @@ // [GenerateResponse], [ExecutionTargetPresence], and the string value types // used by those values. // -// Construction and handle values, including [Config], [Backend], [RunRequest], -// [ArtifactRef], [ExecutionTargetOverride], [Profile], -// [OpenAICompatibleProfileConfig], and [PreparedExecution], do not have stable -// JSON representations. Direct API keys are nevertheless excluded from JSON -// for every public value. +// Construction, inspection, and handle values, including [Config], [Backend], +// [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], +// [OpenAICompatibleProfileConfig], [ProfileInspection], and +// [PreparedExecution], do not have stable JSON representations. Direct API +// keys are nevertheless excluded from JSON for every public value. // // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero. // PreparedRun and RunResult durations are encoded as integer milliseconds in diff --git a/engine.go b/engine.go index c1638df..e5dd3a9 100644 --- a/engine.go +++ b/engine.go @@ -78,14 +78,14 @@ var ( ErrValidation = errors.New("failed to validate output") ) -// Engine prepares and runs Promptkit prompt requests. +// Engine inspects profiles and prepares and runs Promptkit prompt requests. // -// An Engine is safe for concurrent calls to [Engine.Prepare], -// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each -// Engine owns independent backend-capacity pools that coordinate Run and -// RunPrepared admission and model generation. Injected collaborators may still -// be invoked concurrently across different backend pools or for unlimited -// backends. +// An Engine is safe for concurrent calls to [Engine.InspectProfile], +// [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], and +// [Engine.RunPrepared]. Each Engine owns independent backend-capacity pools +// that coordinate Run and RunPrepared admission and model generation. Injected +// collaborators may still be invoked concurrently across different backend +// pools or for unlimited backends. type Engine struct { runner *usecase.Runner } @@ -428,6 +428,46 @@ func fileSource(name string) (fs.FS, string, error) { return os.DirFS(dir), filepath.ToSlash(base), nil } +// InspectProfile resolves one explicit profile without selecting a prompt or +// starting execution work. +// +// InspectProfile trims surrounding whitespace from profileID and looks up the +// resulting nonblank ID exactly and case-sensitively through the engine's +// ordinary in-memory, configured-source, and built-in profile precedence. It +// applies framework defaults, the selected backend, and then the selected +// profile to EffectiveModelParams without a request override. BackendID is +// empty for an endpoint-only profile. +// +// APIKeyEnv in the returned target is an environment-variable name, never its +// value. APIKeyRequired instead reports a direct credential requirement and is +// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives +// an ID from a prompt default_profile nor checks credential availability, so an +// absent or blank named environment variable is not an error. +// +// The returned ProfileInspection and all nested mutable values are +// caller-owned. Filesystem-backed inspection is a point-in-time lookup and +// does not freeze the profile for a later execution. This method does not load +// a prompt, render, read artifacts or schemas, admit backend capacity, contact +// a provider, or generate model output. +// +// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID +// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and +// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend, +// or an invalid resolved target matches ErrProfileLoad. Cancellation during +// profile loading matches ErrProfileLoad while preserving the context error. +// InspectProfile returns no partial result on error. +func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + inspection, err := e.runner.InspectProfile(ctx, profileID) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainProfileInspection(inspection), nil +} + // Prepare resolves and renders a prompt request without calling an LLM. // // Prepare selects the prompt and profile, resolves any selected backend and diff --git a/public_contract_test.go b/public_contract_test.go index c4efeb8..8461f25 100644 --- a/public_contract_test.go +++ b/public_contract_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "reflect" "strings" "sync" @@ -16,6 +17,163 @@ import ( "gitea.maximumdirect.net/eric/promptkit" ) +type inspectionCountingFS struct { + opens atomic.Int64 +} + +func (f *inspectionCountingFS) Open(string) (fs.File, error) { + f.opens.Add(1) + return nil, fs.ErrNotExist +} + +func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) { + const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY" + t.Setenv(environmentName, "") + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(fstest.MapFS{}, "."), + promptkit.WithProfileFS(fstest.MapFS{ + "environment.yaml": &fstest.MapFile{Data: []byte(`id: environment +endpoint: http://environment.example/v1 +model: environment-model +api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY +`)}, + }, "."), + promptkit.WithProfiles( + promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true}, + promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"}, + ), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct inspection engine: %v", err) + } + + for _, tc := range []struct { + profileID string + wantEnv string + wantDirectKey bool + wantEndpoint string + }{ + {profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"}, + {profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"}, + {profileID: "none", wantEndpoint: "http://none.example/v1"}, + } { + t.Run(tc.profileID, func(t *testing.T) { + inspection, err := engine.InspectProfile(context.Background(), tc.profileID) + if err != nil { + t.Fatalf("inspect profile: %v", err) + } + if inspection.ProfileID != strings.TrimSpace(tc.profileID) || + inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint || + inspection.EffectiveModelParams.BackendID != "" || + inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv || + inspection.APIKeyRequired != tc.wantDirectKey { + t.Fatalf("unexpected inspection: %#v", inspection) + } + }) + } + if len(client.requests) != 0 { + t.Fatalf("inspection invoked the model client %d times", len(client.requests)) + } +} + +func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) { + newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine( + promptkit.Config{}, + append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)..., + ) + if err != nil { + t.Fatalf("construct inspection engine: %v", err) + } + return engine + } + + var nilEngine *promptkit.Engine + if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil || + !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err) + } + + valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", + })) + if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err) + } + if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil || + !errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err) + } + + malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{ + "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")}, + }, ".")) + if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil || + !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err) + } + + unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{ + ID: "unknown-backend", BackendID: "unknown", Model: "model", + })) + if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil || + !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err) + } + + countingFS := &inspectionCountingFS{} + canceled := newEngine(t, promptkit.WithProfileFS(countingFS, ".")) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if result, err := canceled.InspectProfile(ctx, "profile"); result != nil || + !errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 { + t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load()) + } +} + +func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) { + extraParams := map[string]any{ + "nested": map[string]any{"value": "original"}, + } + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams, + }), + ) + if err != nil { + t.Fatalf("construct inspection engine: %v", err) + } + + first, err := engine.InspectProfile(context.Background(), "profile") + if err != nil { + t.Fatalf("first inspection: %v", err) + } + first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed" + first.EffectiveModelParams.ExtraParams["later"] = true + + second, err := engine.InspectProfile(context.Background(), "profile") + if err != nil { + t.Fatalf("second inspection: %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatalf("prepare after inspection mutation: %v", err) + } + for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} { + if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil { + t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams) + } + } + if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) { + t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams) + } +} + func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) { payload, err := json.Marshal(promptkit.PreparedRun{}) if err != nil { diff --git a/types.go b/types.go index 2667f89..645c1a4 100644 --- a/types.go +++ b/types.go @@ -320,6 +320,25 @@ type ExecutionTarget struct { ExtraParams map[string]any `json:"extra_params"` } +// ProfileInspection is the caller-owned result of [Engine.InspectProfile]. +// It has no stable JSON representation. +// +// EffectiveModelParams contains a copied effective target. APIKeyRequired is +// separate from that target to preserve ExecutionTarget's general execution +// and stable JSON contracts. +type ProfileInspection struct { + // ProfileID is the trimmed, exact profile ID inspected by the engine. + ProfileID string + // EffectiveModelParams contains framework defaults overlaid by the selected + // backend and then the profile, without a request override. APIKeyEnv is an + // environment-variable name, never its credential value. + EffectiveModelParams ExecutionTarget + // APIKeyRequired reports that a later execution must supply a direct API + // key or an explicit request environment override. It is mutually exclusive + // with a nonblank EffectiveModelParams.APIKeyEnv. + APIKeyRequired bool +} + // ExecutionTargetOverride represents per-request runtime setting overrides and // has no stable JSON representation. //