Expose prompt inspection through the engine

2026-07-30 21:03:47 +00:00
parent 272b6a4bc1
commit e920168b30
5 changed files with 348 additions and 25 deletions
--- a/convert.go
+++ b/convert.go
@@ -181,6 +181,29 @@ func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileI
 	}
 }

+func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
+	if inspection == nil {
+		return nil
+	}
+	inputs := make([]PromptInputDefinition, len(inspection.Inputs))
+	for i, input := range inspection.Inputs {
+		inputs[i] = PromptInputDefinition{
+			Name:        input.Name,
+			Required:    input.Required,
+			ContentType: input.ContentType,
+			Description: input.Description,
+		}
+	}
+	return &PromptInspection{
+		PromptID:         inspection.PromptID,
+		PromptVersion:    inspection.PromptVersion,
+		PromptHash:       inspection.PromptHash,
+		DefaultProfileID: inspection.DefaultProfileID,
+		Inputs:           inputs,
+		OutputContract:   fromDomainOutputContract(inspection.OutputContract),
+	}
+}
+
 func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
 	return ExecutionTargetPresence{
 		Temperature:    presence.Temperature,
--- a/doc.go
+++ b/doc.go
@@ -3,27 +3,27 @@
 //
 // Applications construct an [Engine] with [NewEngine], select filesystem or
 // in-memory sources and optional engine-scoped [Backend] registrations, and
-// call [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
-// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
-// validators, and the built-in OpenAI-compatible client remain internal
-// implementation details.
+// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
+// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
+// registries, repositories, validators, and the built-in OpenAI-compatible
+// client remain internal implementation details.
 //
 // # Concurrency and ownership
 //
-// An Engine supports concurrent InspectProfile, Prepare, PrepareExecution, Run,
-// and RunPrepared calls. Engine-local backend policies bound admitted Run and
-// RunPrepared calls and model generations where configured, while different
-// backend pools and unlimited backends continue independently. An injected
-// [LLMClient] or [ArtifactReader] can therefore still receive concurrent calls
-// and must be safe for that use.
+// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
+// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
+// bound admitted Run and RunPrepared calls and model generations where
+// configured, while different backend pools and unlimited backends continue
+// independently. An injected [LLMClient] or [ArtifactReader] can therefore
+// still receive concurrent calls and must be safe for that use.
 //
 // NewEngine copies in-memory profiles and backend definitions. Prepare,
 // PrepareExecution, and Run copy request maps, slices, pointer values, and
-// JSON-compatible extra parameters before using them. InspectProfile returns
-// copied profile inspection values. Returned values and values passed to
-// extension interfaces are likewise isolated from engine state. Callers own
-// those copies and may mutate them after the call that supplied or returned
-// them.
+// JSON-compatible extra parameters before using them. InspectPrompt and
+// InspectProfile return copied inspection values. Returned values and values
+// passed to extension interfaces are likewise isolated from engine state.
+// Callers own those copies and may mutate them after the call that supplied or
+// returned them.
 //
 // # Security and sensitive data
 //
@@ -51,9 +51,10 @@
 //
 // Construction, inspection, and handle values, including [Config], [Backend],
 // [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
-// [OpenAICompatibleProfileConfig], [ProfileInspection], and
-// [PreparedExecution], do not have stable JSON representations. Direct API
-// keys are nevertheless excluded from JSON for every public value.
+// [OpenAICompatibleProfileConfig], [ProfileInspection],
+// [PromptInputDefinition], [PromptInspection], and [PreparedExecution], do
+// not have stable JSON representations. Direct API keys are nevertheless
+// excluded from JSON for every public value.
 //
 // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
 // PreparedRun and RunResult durations are encoded as integer milliseconds in
--- a/engine.go
+++ b/engine.go
@@ -78,14 +78,15 @@ var (
 	ErrValidation = errors.New("failed to validate output")
 )

-// Engine inspects profiles and prepares and runs Promptkit prompt requests.
+// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
+// requests.
 //
-// An Engine is safe for concurrent calls to [Engine.InspectProfile],
-// [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], and
-// [Engine.RunPrepared]. Each Engine owns independent backend-capacity pools
-// that coordinate Run and RunPrepared admission and model generation. Injected
-// collaborators may still be invoked concurrently across different backend
-// pools or for unlimited backends.
+// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
+// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
+// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
+// backend-capacity pools that coordinate Run and RunPrepared admission and
+// model generation. Injected collaborators may still be invoked concurrently
+// across different backend pools or for unlimited backends.
 type Engine struct {
 	runner *usecase.Runner
 }
@@ -428,6 +429,53 @@ func fileSource(name string) (fs.FS, string, error) {
 	return os.DirFS(dir), filepath.ToSlash(base), nil
 }

+// InspectPrompt resolves one explicit prompt definition without selecting a
+// profile or starting execution work.
+//
+// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
+// promptVersion values unchanged to the engine's ordinary, case-sensitive
+// prompt selection. An empty version succeeds only when that source has one
+// selected ID; a nonempty version selects one exact ID/version pair. The
+// configured prompt source is used without merging, fallback, or enumeration.
+//
+// A successful result proves that the selected definition and any referenced
+// message content files were structurally loaded. Inputs are returned in
+// definition order. DefaultProfileID is declared metadata only and is not
+// resolved. OutputContract is the normalized declared contract, with a JSON
+// Schema path when declared but without loading or compiling that schema.
+// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
+// the selected definition and observed source state; its spelling, length,
+// encoding, algorithm, and security properties are not contracts.
+//
+// This method does not return prompt bodies, templates, source paths, schemas,
+// rendered messages, or execution settings. It does not resolve a profile or
+// credential, read artifacts or schemas, render, validate, admit capacity,
+// contact a provider, or generate model output. The returned PromptInspection
+// and its input slice are caller-owned. Filesystem-backed inspection is a
+// point-in-time lookup and does not freeze a definition for later execution.
+//
+// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
+// matches ErrInvalidRequest. An absent exact ID or version matches
+// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
+// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
+// Cancellation during lookup matches ErrPromptLoad while preserving the
+// context error. InspectPrompt returns no partial result on error.
+func (e *Engine) InspectPrompt(
+	ctx context.Context,
+	promptID string,
+	promptVersion string,
+) (*PromptInspection, error) {
+	if e == nil || e.runner == nil {
+		return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
+	}
+
+	inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
+	if err != nil {
+		return nil, mapPublicError(err)
+	}
+	return fromDomainPromptInspection(inspection), nil
+}
+
 // InspectProfile resolves one explicit profile without selecting a prompt or
 // starting execution work.
 //
--- a/public_contract_test.go
+++ b/public_contract_test.go
@@ -174,6 +174,223 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T)
 	}
 }

+func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
+	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
+	engine, err := promptkit.NewEngine(promptkit.Config{},
+		promptkit.WithPromptFS(fstest.MapFS{
+			"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
+version: "1.0.0"
+messages:
+  - role: user
+    content: old report
+output:
+  format: text
+  validation_mode: none
+`)},
+			"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
+version: "2.0.0"
+default_profile: missing-profile
+inputs:
+  - name: location
+    required: true
+    content_type: text/plain
+    description: Forecast location.
+  - name: units
+    content_type: text/plain
+    description: Unit preference.
+messages:
+  - role: user
+    content_file: messages/report.md
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: schemas/report.json
+`)},
+			"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
+		}, "."),
+		promptkit.WithLLMClient(client),
+	)
+	if err != nil {
+		t.Fatalf("construct prompt-inspection engine: %v", err)
+	}
+
+	inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
+	if err != nil {
+		t.Fatalf("inspect prompt: %v", err)
+	}
+	if inspection.PromptID != "report" ||
+		inspection.PromptVersion != "2.0.0" ||
+		inspection.PromptHash == "" ||
+		inspection.DefaultProfileID != "missing-profile" ||
+		inspection.OutputContract != (promptkit.OutputContract{
+			Format:         promptkit.FormatJSON,
+			ValidationMode: promptkit.ValidationJSONSchema,
+			SchemaPath:     "schemas/report.json",
+		}) {
+		t.Fatalf("unexpected inspection metadata: %#v", inspection)
+	}
+	wantInputs := []promptkit.PromptInputDefinition{
+		{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
+		{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
+	}
+	if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
+		t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
+	}
+	if len(client.requests) != 0 {
+		t.Fatalf("inspection invoked the model client %d times", len(client.requests))
+	}
+}
+
+func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
+	newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
+		t.Helper()
+		engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
+		if err != nil {
+			t.Fatalf("construct prompt-inspection engine: %v", err)
+		}
+		return engine
+	}
+	validSource := fstest.MapFS{
+		"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
+version: "1"
+messages:
+  - role: user
+    content: body
+output:
+  format: text
+  validation_mode: none
+`)},
+	}
+
+	var nilEngine *promptkit.Engine
+	if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
+		!errors.Is(err, promptkit.ErrInvalidConfig) {
+		t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
+	}
+
+	valid := newEngine(t, validSource)
+	if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
+		!errors.Is(err, promptkit.ErrInvalidRequest) {
+		t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
+	}
+	if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
+		!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
+		t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
+	}
+	if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
+		!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
+		t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
+	}
+
+	ambiguous := newEngine(t, fstest.MapFS{
+		"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
+version: "1"
+messages:
+  - role: user
+    content: first
+output:
+  format: text
+  validation_mode: none
+`)},
+		"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
+version: "2"
+messages:
+  - role: user
+    content: second
+output:
+  format: text
+  validation_mode: none
+`)},
+	})
+	if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
+		!errors.Is(err, promptkit.ErrPromptLoad) {
+		t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
+	}
+
+	for name, source := range map[string]fstest.MapFS{
+		"malformed definition": {
+			"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
+		},
+		"missing content file": {
+			"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
+version: "1"
+messages:
+  - role: user
+    content_file: missing.md
+output:
+  format: text
+  validation_mode: none
+`)},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
+				!errors.Is(err, promptkit.ErrPromptLoad) {
+				t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
+			}
+		})
+	}
+
+	countingFS := &inspectionCountingFS{}
+	canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
+	if err != nil {
+		t.Fatalf("construct canceled prompt-inspection engine: %v", err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
+		!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
+		t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
+	}
+}
+
+func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
+	engine, err := promptkit.NewEngine(promptkit.Config{},
+		promptkit.WithPromptFS(fstest.MapFS{
+			"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
+version: "1"
+default_profile: profile
+inputs:
+  - name: subject
+    content_type: text/plain
+    description: Summary subject.
+messages:
+  - role: user
+    content: summarize
+output:
+  format: markdown
+  validation_mode: basic
+`)},
+		}, "."),
+		promptkit.WithProfiles(promptkit.Profile{
+			ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
+		}),
+	)
+	if err != nil {
+		t.Fatalf("construct prompt-inspection engine: %v", err)
+	}
+
+	first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
+	if err != nil {
+		t.Fatalf("first inspection: %v", err)
+	}
+	first.Inputs[0].Name = "changed"
+	first.OutputContract.SchemaPath = "changed.json"
+
+	second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
+	if err != nil {
+		t.Fatalf("second inspection: %v", err)
+	}
+	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
+	if err != nil {
+		t.Fatalf("prepare after inspection mutation: %v", err)
+	}
+	if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
+		prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
+		t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
+	}
+}
+
 func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
 	payload, err := json.Marshal(promptkit.PreparedRun{})
 	if err != nil {
--- a/types.go
+++ b/types.go
@@ -339,6 +339,40 @@ type ProfileInspection struct {
 	APIKeyRequired bool
 }

+// PromptInputDefinition describes one declared prompt input.
+// It has no stable JSON representation.
+type PromptInputDefinition struct {
+	// Name is the normalized prompt input name.
+	Name string
+	// Required reports whether preparation requires the input.
+	Required bool
+	// ContentType is the declared input media-type metadata.
+	ContentType string
+	// Description is the declared human-readable input description.
+	Description string
+}
+
+// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
+// It has no stable JSON representation.
+//
+// Inputs contains copied declared input metadata in definition order.
+// OutputContract is the normalized contract declared by the prompt definition,
+// rather than a request-level effective override. PromptHash is opaque.
+type PromptInspection struct {
+	// PromptID is the normalized ID of the selected prompt definition.
+	PromptID string
+	// PromptVersion is the normalized version of the selected prompt definition.
+	PromptVersion string
+	// PromptHash is the opaque equality value for the selected definition.
+	PromptHash string
+	// DefaultProfileID is declared metadata and is not resolved by inspection.
+	DefaultProfileID string
+	// Inputs contains caller-owned declared input metadata in definition order.
+	Inputs []PromptInputDefinition
+	// OutputContract is the normalized contract declared by the definition.
+	OutputContract OutputContract
+}
+
 // ExecutionTargetOverride represents per-request runtime setting overrides and
 // has no stable JSON representation.
 //