diff --git a/convert.go b/convert.go index daf05dc..0b75d9d 100644 --- a/convert.go +++ b/convert.go @@ -181,6 +181,29 @@ func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileI } } +func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection { + if inspection == nil { + return nil + } + inputs := make([]PromptInputDefinition, len(inspection.Inputs)) + for i, input := range inspection.Inputs { + inputs[i] = PromptInputDefinition{ + Name: input.Name, + Required: input.Required, + ContentType: input.ContentType, + Description: input.Description, + } + } + return &PromptInspection{ + PromptID: inspection.PromptID, + PromptVersion: inspection.PromptVersion, + PromptHash: inspection.PromptHash, + DefaultProfileID: inspection.DefaultProfileID, + Inputs: inputs, + OutputContract: fromDomainOutputContract(inspection.OutputContract), + } +} + func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence { return ExecutionTargetPresence{ Temperature: presence.Temperature, diff --git a/doc.go b/doc.go index c5e88c2..a7206a0 100644 --- a/doc.go +++ b/doc.go @@ -3,27 +3,27 @@ // // Applications construct an [Engine] with [NewEngine], select filesystem or // in-memory sources and optional engine-scoped [Backend] registrations, and -// call [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution], -// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories, -// validators, and the built-in OpenAI-compatible client remain internal -// implementation details. +// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare], +// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete +// registries, repositories, validators, and the built-in OpenAI-compatible +// client remain internal implementation details. // // # Concurrency and ownership // -// An Engine supports concurrent InspectProfile, Prepare, PrepareExecution, Run, -// and RunPrepared calls. Engine-local backend policies bound admitted Run and -// RunPrepared calls and model generations where configured, while different -// backend pools and unlimited backends continue independently. An injected -// [LLMClient] or [ArtifactReader] can therefore still receive concurrent calls -// and must be safe for that use. +// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare, +// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies +// bound admitted Run and RunPrepared calls and model generations where +// configured, while different backend pools and unlimited backends continue +// independently. An injected [LLMClient] or [ArtifactReader] can therefore +// still receive concurrent calls and must be safe for that use. // // NewEngine copies in-memory profiles and backend definitions. Prepare, // PrepareExecution, and Run copy request maps, slices, pointer values, and -// JSON-compatible extra parameters before using them. InspectProfile returns -// copied profile inspection values. Returned values and values passed to -// extension interfaces are likewise isolated from engine state. Callers own -// those copies and may mutate them after the call that supplied or returned -// them. +// JSON-compatible extra parameters before using them. InspectPrompt and +// InspectProfile return copied inspection values. Returned values and values +// passed to extension interfaces are likewise isolated from engine state. +// Callers own those copies and may mutate them after the call that supplied or +// returned them. // // # Security and sensitive data // @@ -51,9 +51,10 @@ // // Construction, inspection, and handle values, including [Config], [Backend], // [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], -// [OpenAICompatibleProfileConfig], [ProfileInspection], and -// [PreparedExecution], do not have stable JSON representations. Direct API -// keys are nevertheless excluded from JSON for every public value. +// [OpenAICompatibleProfileConfig], [ProfileInspection], +// [PromptInputDefinition], [PromptInspection], and [PreparedExecution], do +// not have stable JSON representations. Direct API keys are nevertheless +// excluded from JSON for every public value. // // JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero. // PreparedRun and RunResult durations are encoded as integer milliseconds in diff --git a/engine.go b/engine.go index e5dd3a9..6614f3f 100644 --- a/engine.go +++ b/engine.go @@ -78,14 +78,15 @@ var ( ErrValidation = errors.New("failed to validate output") ) -// Engine inspects profiles and prepares and runs Promptkit prompt requests. +// Engine inspects prompts and profiles and prepares and runs Promptkit prompt +// requests. // -// An Engine is safe for concurrent calls to [Engine.InspectProfile], -// [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], and -// [Engine.RunPrepared]. Each Engine owns independent backend-capacity pools -// that coordinate Run and RunPrepared admission and model generation. Injected -// collaborators may still be invoked concurrently across different backend -// pools or for unlimited backends. +// An Engine is safe for concurrent calls to [Engine.InspectPrompt], +// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution], +// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent +// backend-capacity pools that coordinate Run and RunPrepared admission and +// model generation. Injected collaborators may still be invoked concurrently +// across different backend pools or for unlimited backends. type Engine struct { runner *usecase.Runner } @@ -428,6 +429,53 @@ func fileSource(name string) (fs.FS, string, error) { return os.DirFS(dir), filepath.ToSlash(base), nil } +// InspectPrompt resolves one explicit prompt definition without selecting a +// profile or starting execution work. +// +// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and +// promptVersion values unchanged to the engine's ordinary, case-sensitive +// prompt selection. An empty version succeeds only when that source has one +// selected ID; a nonempty version selects one exact ID/version pair. The +// configured prompt source is used without merging, fallback, or enumeration. +// +// A successful result proves that the selected definition and any referenced +// message content files were structurally loaded. Inputs are returned in +// definition order. DefaultProfileID is declared metadata only and is not +// resolved. OutputContract is the normalized declared contract, with a JSON +// Schema path when declared but without loading or compiling that schema. +// PromptHash is the same opaque equality value as PreparedRun.PromptHash for +// the selected definition and observed source state; its spelling, length, +// encoding, algorithm, and security properties are not contracts. +// +// This method does not return prompt bodies, templates, source paths, schemas, +// rendered messages, or execution settings. It does not resolve a profile or +// credential, read artifacts or schemas, render, validate, admit capacity, +// contact a provider, or generate model output. The returned PromptInspection +// and its input slice are caller-owned. Filesystem-backed inspection is a +// point-in-time lookup and does not freeze a definition for later execution. +// +// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID +// matches ErrInvalidRequest. An absent exact ID or version matches +// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate, +// ambiguous, referenced-content, or hashing failures match ErrPromptLoad. +// Cancellation during lookup matches ErrPromptLoad while preserving the +// context error. InspectPrompt returns no partial result on error. +func (e *Engine) InspectPrompt( + ctx context.Context, + promptID string, + promptVersion string, +) (*PromptInspection, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainPromptInspection(inspection), nil +} + // InspectProfile resolves one explicit profile without selecting a prompt or // starting execution work. // diff --git a/public_contract_test.go b/public_contract_test.go index 8461f25..96d4114 100644 --- a/public_contract_test.go +++ b/public_contract_test.go @@ -174,6 +174,223 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) } } +func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(fstest.MapFS{ + "report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report +version: "1.0.0" +messages: + - role: user + content: old report +output: + format: text + validation_mode: none +`)}, + "report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report +version: "2.0.0" +default_profile: missing-profile +inputs: + - name: location + required: true + content_type: text/plain + description: Forecast location. + - name: units + content_type: text/plain + description: Unit preference. +messages: + - role: user + content_file: messages/report.md +output: + format: json + validation_mode: json_schema + schema_path: schemas/report.json +`)}, + "messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")}, + }, "."), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct prompt-inspection engine: %v", err) + } + + inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0") + if err != nil { + t.Fatalf("inspect prompt: %v", err) + } + if inspection.PromptID != "report" || + inspection.PromptVersion != "2.0.0" || + inspection.PromptHash == "" || + inspection.DefaultProfileID != "missing-profile" || + inspection.OutputContract != (promptkit.OutputContract{ + Format: promptkit.FormatJSON, + ValidationMode: promptkit.ValidationJSONSchema, + SchemaPath: "schemas/report.json", + }) { + t.Fatalf("unexpected inspection metadata: %#v", inspection) + } + wantInputs := []promptkit.PromptInputDefinition{ + {Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."}, + {Name: "units", ContentType: "text/plain", Description: "Unit preference."}, + } + if !reflect.DeepEqual(inspection.Inputs, wantInputs) { + t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs) + } + if len(client.requests) != 0 { + t.Fatalf("inspection invoked the model client %d times", len(client.requests)) + } +} + +func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) { + newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, ".")) + if err != nil { + t.Fatalf("construct prompt-inspection engine: %v", err) + } + return engine + } + validSource := fstest.MapFS{ + "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt +version: "1" +messages: + - role: user + content: body +output: + format: text + validation_mode: none +`)}, + } + + var nilEngine *promptkit.Engine + if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil || + !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err) + } + + valid := newEngine(t, validSource) + if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err) + } + if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil || + !errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err) + } + if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil || + !errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err) + } + + ambiguous := newEngine(t, fstest.MapFS{ + "one.yaml": &fstest.MapFile{Data: []byte(`id: prompt +version: "1" +messages: + - role: user + content: first +output: + format: text + validation_mode: none +`)}, + "two.yaml": &fstest.MapFile{Data: []byte(`id: prompt +version: "2" +messages: + - role: user + content: second +output: + format: text + validation_mode: none +`)}, + }) + if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil || + !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err) + } + + for name, source := range map[string]fstest.MapFS{ + "malformed definition": { + "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")}, + }, + "missing content file": { + "broken.yaml": &fstest.MapFile{Data: []byte(`id: broken +version: "1" +messages: + - role: user + content_file: missing.md +output: + format: text + validation_mode: none +`)}, + }, + } { + t.Run(name, func(t *testing.T) { + if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil || + !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err) + } + }) + } + + countingFS := &inspectionCountingFS{} + canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, ".")) + if err != nil { + t.Fatalf("construct canceled prompt-inspection engine: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil || + !errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 { + t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load()) + } +} + +func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(fstest.MapFS{ + "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt +version: "1" +default_profile: profile +inputs: + - name: subject + content_type: text/plain + description: Summary subject. +messages: + - role: user + content: summarize +output: + format: markdown + validation_mode: basic +`)}, + }, "."), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", + }), + ) + if err != nil { + t.Fatalf("construct prompt-inspection engine: %v", err) + } + + first, err := engine.InspectPrompt(context.Background(), "prompt", "1") + if err != nil { + t.Fatalf("first inspection: %v", err) + } + first.Inputs[0].Name = "changed" + first.OutputContract.SchemaPath = "changed.json" + + second, err := engine.InspectPrompt(context.Background(), "prompt", "1") + if err != nil { + t.Fatalf("second inspection: %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"}) + if err != nil { + t.Fatalf("prepare after inspection mutation: %v", err) + } + if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" || + prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash { + t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared) + } +} + func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) { payload, err := json.Marshal(promptkit.PreparedRun{}) if err != nil { diff --git a/types.go b/types.go index 645c1a4..9e90188 100644 --- a/types.go +++ b/types.go @@ -339,6 +339,40 @@ type ProfileInspection struct { APIKeyRequired bool } +// PromptInputDefinition describes one declared prompt input. +// It has no stable JSON representation. +type PromptInputDefinition struct { + // Name is the normalized prompt input name. + Name string + // Required reports whether preparation requires the input. + Required bool + // ContentType is the declared input media-type metadata. + ContentType string + // Description is the declared human-readable input description. + Description string +} + +// PromptInspection is the caller-owned result of [Engine.InspectPrompt]. +// It has no stable JSON representation. +// +// Inputs contains copied declared input metadata in definition order. +// OutputContract is the normalized contract declared by the prompt definition, +// rather than a request-level effective override. PromptHash is opaque. +type PromptInspection struct { + // PromptID is the normalized ID of the selected prompt definition. + PromptID string + // PromptVersion is the normalized version of the selected prompt definition. + PromptVersion string + // PromptHash is the opaque equality value for the selected definition. + PromptHash string + // DefaultProfileID is declared metadata and is not resolved by inspection. + DefaultProfileID string + // Inputs contains caller-owned declared input metadata in definition order. + Inputs []PromptInputDefinition + // OutputContract is the normalized contract declared by the definition. + OutputContract OutputContract +} + // ExecutionTargetOverride represents per-request runtime setting overrides and // has no stable JSON representation. //