Expose profile inspection through the engine

2026-07-30 19:52:00 +00:00
parent 0bf5f88136
commit 242eace4a7
5 changed files with 255 additions and 25 deletions
--- a/convert.go
+++ b/convert.go
@@ -170,6 +170,17 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
 	}
 }

+func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
+	if inspection == nil {
+		return nil
+	}
+	return &ProfileInspection{
+		ProfileID:            inspection.ProfileID,
+		EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
+		APIKeyRequired:       inspection.APIKeyRequired,
+	}
+}
+
 func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
 	return ExecutionTargetPresence{
 		Temperature:    presence.Temperature,
--- a/doc.go
+++ b/doc.go
@@ -3,25 +3,27 @@
 //
 // Applications construct an [Engine] with [NewEngine], select filesystem or
 // in-memory sources and optional engine-scoped [Backend] registrations, and
-// call [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], or
-// [Engine.RunPrepared]. Concrete registries, repositories, validators, and the
-// built-in OpenAI-compatible client remain internal implementation details.
+// call [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
+// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
+// validators, and the built-in OpenAI-compatible client remain internal
+// implementation details.
 //
 // # Concurrency and ownership
 //
-// An Engine supports concurrent Prepare, PrepareExecution, Run, and RunPrepared
-// calls. Engine-local backend policies bound admitted Run and RunPrepared calls
-// and model generations where configured, while different backend pools and
-// unlimited backends continue independently. An injected [LLMClient] or
-// [ArtifactReader] can therefore still receive concurrent calls and must be
-// safe for that use.
+// An Engine supports concurrent InspectProfile, Prepare, PrepareExecution, Run,
+// and RunPrepared calls. Engine-local backend policies bound admitted Run and
+// RunPrepared calls and model generations where configured, while different
+// backend pools and unlimited backends continue independently. An injected
+// [LLMClient] or [ArtifactReader] can therefore still receive concurrent calls
+// and must be safe for that use.
 //
 // NewEngine copies in-memory profiles and backend definitions. Prepare,
 // PrepareExecution, and Run copy request maps, slices, pointer values, and
-// JSON-compatible extra parameters before using them. Returned values and
-// values passed to extension interfaces are likewise isolated from engine
-// state. Callers own those copies and may mutate them after the call that
-// supplied or returned them.
+// JSON-compatible extra parameters before using them. InspectProfile returns
+// copied profile inspection values. Returned values and values passed to
+// extension interfaces are likewise isolated from engine state. Callers own
+// those copies and may mutate them after the call that supplied or returned
+// them.
 //
 // # Security and sensitive data
 //
@@ -47,11 +49,11 @@
 // [GenerateResponse], [ExecutionTargetPresence], and the string value types
 // used by those values.
 //
-// Construction and handle values, including [Config], [Backend], [RunRequest],
-// [ArtifactRef], [ExecutionTargetOverride], [Profile],
-// [OpenAICompatibleProfileConfig], and [PreparedExecution], do not have stable
-// JSON representations. Direct API keys are nevertheless excluded from JSON
-// for every public value.
+// Construction, inspection, and handle values, including [Config], [Backend],
+// [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
+// [OpenAICompatibleProfileConfig], [ProfileInspection], and
+// [PreparedExecution], do not have stable JSON representations. Direct API
+// keys are nevertheless excluded from JSON for every public value.
 //
 // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
 // PreparedRun and RunResult durations are encoded as integer milliseconds in
--- a/engine.go
+++ b/engine.go
@@ -78,14 +78,14 @@ var (
 	ErrValidation = errors.New("failed to validate output")
 )

-// Engine prepares and runs Promptkit prompt requests.
+// Engine inspects profiles and prepares and runs Promptkit prompt requests.
 //
-// An Engine is safe for concurrent calls to [Engine.Prepare],
-// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each
-// Engine owns independent backend-capacity pools that coordinate Run and
-// RunPrepared admission and model generation. Injected collaborators may still
-// be invoked concurrently across different backend pools or for unlimited
-// backends.
+// An Engine is safe for concurrent calls to [Engine.InspectProfile],
+// [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], and
+// [Engine.RunPrepared]. Each Engine owns independent backend-capacity pools
+// that coordinate Run and RunPrepared admission and model generation. Injected
+// collaborators may still be invoked concurrently across different backend
+// pools or for unlimited backends.
 type Engine struct {
 	runner *usecase.Runner
 }
@@ -428,6 +428,46 @@ func fileSource(name string) (fs.FS, string, error) {
 	return os.DirFS(dir), filepath.ToSlash(base), nil
 }

+// InspectProfile resolves one explicit profile without selecting a prompt or
+// starting execution work.
+//
+// InspectProfile trims surrounding whitespace from profileID and looks up the
+// resulting nonblank ID exactly and case-sensitively through the engine's
+// ordinary in-memory, configured-source, and built-in profile precedence. It
+// applies framework defaults, the selected backend, and then the selected
+// profile to EffectiveModelParams without a request override. BackendID is
+// empty for an endpoint-only profile.
+//
+// APIKeyEnv in the returned target is an environment-variable name, never its
+// value. APIKeyRequired instead reports a direct credential requirement and is
+// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
+// an ID from a prompt default_profile nor checks credential availability, so an
+// absent or blank named environment variable is not an error.
+//
+// The returned ProfileInspection and all nested mutable values are
+// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
+// does not freeze the profile for a later execution. This method does not load
+// a prompt, render, read artifacts or schemas, admit backend capacity, contact
+// a provider, or generate model output.
+//
+// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
+// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
+// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
+// or an invalid resolved target matches ErrProfileLoad. Cancellation during
+// profile loading matches ErrProfileLoad while preserving the context error.
+// InspectProfile returns no partial result on error.
+func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
+	if e == nil || e.runner == nil {
+		return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
+	}
+
+	inspection, err := e.runner.InspectProfile(ctx, profileID)
+	if err != nil {
+		return nil, mapPublicError(err)
+	}
+	return fromDomainProfileInspection(inspection), nil
+}
+
 // Prepare resolves and renders a prompt request without calling an LLM.
 //
 // Prepare selects the prompt and profile, resolves any selected backend and
--- a/public_contract_test.go
+++ b/public_contract_test.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"reflect"
 	"strings"
 	"sync"
@@ -16,6 +17,163 @@ import (
 	"gitea.maximumdirect.net/eric/promptkit"
 )

+type inspectionCountingFS struct {
+	opens atomic.Int64
+}
+
+func (f *inspectionCountingFS) Open(string) (fs.File, error) {
+	f.opens.Add(1)
+	return nil, fs.ErrNotExist
+}
+
+func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
+	const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
+	t.Setenv(environmentName, "")
+	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
+	engine, err := promptkit.NewEngine(promptkit.Config{},
+		promptkit.WithPromptFS(fstest.MapFS{}, "."),
+		promptkit.WithProfileFS(fstest.MapFS{
+			"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
+endpoint: http://environment.example/v1
+model: environment-model
+api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
+`)},
+		}, "."),
+		promptkit.WithProfiles(
+			promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
+			promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
+		),
+		promptkit.WithLLMClient(client),
+	)
+	if err != nil {
+		t.Fatalf("construct inspection engine: %v", err)
+	}
+
+	for _, tc := range []struct {
+		profileID     string
+		wantEnv       string
+		wantDirectKey bool
+		wantEndpoint  string
+	}{
+		{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
+		{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
+		{profileID: "none", wantEndpoint: "http://none.example/v1"},
+	} {
+		t.Run(tc.profileID, func(t *testing.T) {
+			inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
+			if err != nil {
+				t.Fatalf("inspect profile: %v", err)
+			}
+			if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
+				inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
+				inspection.EffectiveModelParams.BackendID != "" ||
+				inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
+				inspection.APIKeyRequired != tc.wantDirectKey {
+				t.Fatalf("unexpected inspection: %#v", inspection)
+			}
+		})
+	}
+	if len(client.requests) != 0 {
+		t.Fatalf("inspection invoked the model client %d times", len(client.requests))
+	}
+}
+
+func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
+	newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
+		t.Helper()
+		engine, err := promptkit.NewEngine(
+			promptkit.Config{},
+			append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
+		)
+		if err != nil {
+			t.Fatalf("construct inspection engine: %v", err)
+		}
+		return engine
+	}
+
+	var nilEngine *promptkit.Engine
+	if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
+		!errors.Is(err, promptkit.ErrInvalidConfig) {
+		t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
+	}
+
+	valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
+		ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
+	}))
+	if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
+		!errors.Is(err, promptkit.ErrInvalidRequest) {
+		t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
+	}
+	if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
+		!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
+		t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
+	}
+
+	malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
+		"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")},
+	}, "."))
+	if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
+		!errors.Is(err, promptkit.ErrProfileLoad) {
+		t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
+	}
+
+	unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
+		ID: "unknown-backend", BackendID: "unknown", Model: "model",
+	}))
+	if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
+		!errors.Is(err, promptkit.ErrProfileLoad) {
+		t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
+	}
+
+	countingFS := &inspectionCountingFS{}
+	canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
+		!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
+		t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
+	}
+}
+
+func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
+	extraParams := map[string]any{
+		"nested": map[string]any{"value": "original"},
+	}
+	engine, err := promptkit.NewEngine(promptkit.Config{},
+		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
+		promptkit.WithProfiles(promptkit.Profile{
+			ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
+		}),
+	)
+	if err != nil {
+		t.Fatalf("construct inspection engine: %v", err)
+	}
+
+	first, err := engine.InspectProfile(context.Background(), "profile")
+	if err != nil {
+		t.Fatalf("first inspection: %v", err)
+	}
+	first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
+	first.EffectiveModelParams.ExtraParams["later"] = true
+
+	second, err := engine.InspectProfile(context.Background(), "profile")
+	if err != nil {
+		t.Fatalf("second inspection: %v", err)
+	}
+	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
+	if err != nil {
+		t.Fatalf("prepare after inspection mutation: %v", err)
+	}
+	for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
+		if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
+			t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
+		}
+	}
+	if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
+		t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
+	}
+}
+
 func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
 	payload, err := json.Marshal(promptkit.PreparedRun{})
 	if err != nil {
--- a/types.go
+++ b/types.go
@@ -320,6 +320,25 @@ type ExecutionTarget struct {
 	ExtraParams map[string]any `json:"extra_params"`
 }

+// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
+// It has no stable JSON representation.
+//
+// EffectiveModelParams contains a copied effective target. APIKeyRequired is
+// separate from that target to preserve ExecutionTarget's general execution
+// and stable JSON contracts.
+type ProfileInspection struct {
+	// ProfileID is the trimmed, exact profile ID inspected by the engine.
+	ProfileID string
+	// EffectiveModelParams contains framework defaults overlaid by the selected
+	// backend and then the profile, without a request override. APIKeyEnv is an
+	// environment-variable name, never its credential value.
+	EffectiveModelParams ExecutionTarget
+	// APIKeyRequired reports that a later execution must supply a direct API
+	// key or an explicit request environment override. It is mutually exclusive
+	// with a nonblank EffectiveModelParams.APIKeyEnv.
+	APIKeyRequired bool
+}
+
 // ExecutionTargetOverride represents per-request runtime setting overrides and
 // has no stable JSON representation.
 //