From 272b6a4bc17c203fc9f85d307bec060cb562a861 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 21:00:05 +0000 Subject: [PATCH] Add internal prompt inspection --- docs/roadmap/future.md | 16 +- docs/roadmap/implementation.md | 782 +++++++++--------- docs/roadmap/prompt-inspection.md | 268 ++++++ .../weatherreporter-promptkit-wishlist.md | 2 +- internal/domain/domain.go | 10 + internal/usecase/prompt_inspection.go | 73 ++ internal/usecase/prompt_inspection_test.go | 157 ++++ internal/usecase/runner.go | 10 +- 8 files changed, 919 insertions(+), 399 deletions(-) create mode 100644 docs/roadmap/prompt-inspection.md create mode 100644 internal/usecase/prompt_inspection.go create mode 100644 internal/usecase/prompt_inspection_test.go diff --git a/docs/roadmap/future.md b/docs/roadmap/future.md index 6e9dd64..ba8aece 100644 --- a/docs/roadmap/future.md +++ b/docs/roadmap/future.md @@ -33,19 +33,9 @@ consumers. ## Ideas -### Prompt-definition inspection - -Provide exact prompt-definition lookup without rendering, placeholder inputs, -profile resolution, or model generation, as requested by -[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-2-prompt-definition-inspection). - -- Return caller-owned identity, version, input definitions, default-profile, - output-contract, and opaque definition-equality information. -- Apply ordinary prompt-source precedence and exact ID/version selection. -- Validate the selected definition and referenced prompt content - structurally, without returning source bodies or rendered messages. -- Leave complete cross-source corpus validation and enumeration outside the - initial inspection contract. +Prompt-definition inspection has been selected for active planning in the +[focused feature roadmap](prompt-inspection.md). The remaining idea is still +available for future selection. ### Structured capacity errors diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a3f9e5b..4e22d5d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,12 +1,12 @@ -# Prompt-Independent Profile Inspection Implementation Plan +# Prompt-Definition Inspection Implementation Plan -**Status:** Complete. +**Status:** Accepted. ## Purpose This document is the decision-complete implementation plan for -[prompt-independent profile inspection](profile-inspection.md). It is written -for a coding agent that will implement each stage in order. +[prompt-definition inspection](prompt-inspection.md). It is written for a +coding agent that will implement each stage in order. The feature roadmap owns the motivation, consumer workflow, policy choices, compatibility requirements, non-goals, and target end state. This document @@ -18,468 +18,488 @@ ownership, documentation work, validation commands, and completion gates. - Complete the stages in order. Keep the repository compiling and the focused tests passing at every stage boundary. - Preserve unrelated working-tree changes. In particular, retain the accepted - feature roadmap and the corresponding future-roadmap and downstream-wishlist - edits that may already be uncommitted. + feature roadmap and the corresponding future-catalog and Weatherreporter + wishlist edits that may already be uncommitted. - Follow every policy under `docs/policy/`, the task-specific reading guide in `docs/development.md`, and the accepted behavior in - `profile-inspection.md`. -- Keep the supported API in the root `promptkit` package. Profile repositories, - backend resolution, and target assembly remain below Go's `internal/` - boundary. -- Reuse the exact profile-source, backend-membership, and execution-target - precedence used by `Prepare`, `PrepareExecution`, and `Run`. Do not create a - second profile loader, backend registry, or target-merging implementation. -- Preserve the observable behavior and error ordering of existing preparation - and execution methods. The new inspection operation must not require a - prompt, prompt default profile, request override, credential value, capacity - admission, model client, renderer, artifact reader, schema source, or - validator. -- Treat structural validation as the existing profile, backend, and effective - target invariants. Do not add new endpoint URL policy, provider - connectivity checks, reserved-extra-parameter policy, credential syntax, or - file-format validation rules as part of this feature. -- Never read, retain, return, format, or log an environment credential value. - The result may contain only the effective environment-variable name and the - direct-key-required boolean. -- Return caller-owned public values. In particular, nested - `ExecutionTarget.ExtraParams` maps, slices, and objects must not alias - engine-owned state or another inspection result. -- Keep tests lean and behavior-focused. Reuse existing profile repository, - backend registry, target precedence, and preparation tests rather than - duplicating their complete matrices. + `prompt-inspection.md`. +- Keep the supported API in the root `promptkit` package. Prompt repositories + and internal prompt definitions remain below Go's `internal/` boundary. +- Reuse the exact prompt source, ID/version selection, definition + normalization, referenced-content loading, and prompt hashing used by + `Prepare`, `PrepareExecution`, and `Run`. Do not create a second parser, + source abstraction, or hash algorithm. +- Preserve the observable validation order, error identities, and results of + existing preparation and execution methods. +- The new inspection operation must not require a profile, backend, request + override, credential, artifact, schema source, renderer, validator, + capacity admission, output repairer, or model client. +- Treat structural validation as the existing prompt-repository contract. + Do not add template parsing, input media-type enforcement, schema loading, + schema compilation, profile validation, source enumeration, or provider + connectivity checks. +- Return only the metadata accepted by the feature roadmap. Do not expose raw + YAML, prompt descriptions, source paths, message or session templates, + cache-control declarations, rendered content, or schema bodies. +- Return caller-owned public values. The input-definition slice must not alias + repository, runner, engine, or another result's state. +- Keep tests lean and behavior-focused. Reuse existing prompt-repository and + preparation coverage rather than duplicating their complete format and + source matrices. - Update exact contracts in GoDoc with the exported declarations. Update current-state consumer, format, and internal documentation only after the corresponding code exists. -- Do not add release notes, change a module version, create a release, or tag a - commit as part of this work. +- Do not add release notes, change a module version, create a release, commit, + or tag as part of this work. ## Fixed Design ### Public API -Add this root-package value immediately after `ExecutionTarget` in `types.go`: +Add these root-package values in `types.go` immediately after +`ProfileInspection`: ```go -// ProfileInspection is the caller-owned result of Engine.InspectProfile. -// The exact GoDoc is specified below. -type ProfileInspection struct { - ProfileID string - EffectiveModelParams ExecutionTarget - APIKeyRequired bool +type PromptInputDefinition struct { + Name string + Required bool + ContentType string + Description string +} + +type PromptInspection struct { + PromptID string + PromptVersion string + PromptHash string + DefaultProfileID string + Inputs []PromptInputDefinition + OutputContract OutputContract } ``` -Add this method to `Engine` in `engine.go`, immediately before `Prepare`: +Add this method to `Engine` in `engine.go`, immediately before +`InspectProfile`: ```go -func (e *Engine) InspectProfile( +func (e *Engine) InspectPrompt( ctx context.Context, - profileID string, -) (*ProfileInspection, error) + promptID string, + promptVersion string, +) (*PromptInspection, error) ``` -Do not accept `RunRequest`, `ExecutionTargetOverride`, an API key, an -environment override, or options on this method. Exact inspection of one -explicit profile ID is the complete operation. +Do not accept a `RunRequest`, inputs, variables, profile ID, output override, +or options on this method. Exact inspection of one prompt ID and optional +version is the complete operation. -`ProfileInspection` has no stable JSON contract. Do not add JSON tags, -`MarshalJSON`, `UnmarshalJSON`, or a custom string representation. Ordinary Go -encoding of the exported fields is not prohibited, but consumers must not be -promised compatibility for that encoding. +`PromptInputDefinition` and `PromptInspection` have no stable JSON contract. +Do not add JSON tags, `MarshalJSON`, `UnmarshalJSON`, or custom string +representations. Their nested `OutputContract` continues to use its existing +stable JSON representation when encoded independently, but that does not make +the enclosing inspection value stable. The exact type and method GoDoc must establish: -- surrounding whitespace is trimmed from `profileID`; the resulting nonblank - ID is looked up exactly and case-sensitively; -- the result applies the engine's ordinary in-memory, configured-source, and - built-in profile precedence; -- `EffectiveModelParams` contains framework defaults overlaid by the selected - backend and then the selected profile, with no request override; -- `EffectiveModelParams.BackendID` is empty for endpoint-only profiles; -- `EffectiveModelParams.APIKeyEnv` is an environment-variable name and never - its value; -- `APIKeyRequired` means a direct request credential is required and is - mutually exclusive with a nonblank effective `APIKeyEnv`; -- the method never derives an ID from a prompt's `default_profile`; -- the method performs no prompt lookup, rendering, artifact or schema work, - backend admission, provider connectivity check, or model generation; -- credential availability is not checked, so an absent or blank named - environment variable is not an error; -- the returned value and all nested mutable values are caller-owned; +- `promptID` is required and whitespace-only is invalid; +- nonblank `promptID` and `promptVersion` values are passed to ordinary prompt + selection unchanged rather than trimmed or otherwise canonicalized; +- lookup is case-sensitive and exact; +- an empty version succeeds only when exactly one definition has the selected + ID, while a nonempty version selects one exact ID/version pair; +- the engine's configured prompt source and existing source-option selection + are used without merging, fallback, or enumeration; +- a successful result proves that the selected definition and referenced + message content files were structurally loaded through the ordinary prompt + repository; +- input metadata is returned in definition order and includes name, required + status, content type, and description; +- `DefaultProfileID` is declared metadata only and is not resolved; +- `OutputContract` is the normalized declared contract, not a request-level + effective override; +- a JSON Schema path is returned when declared, but the schema and its + references are not loaded or compiled; +- `PromptHash` is the same opaque equality value used by + `PreparedRun.PromptHash` for the same selected definition and observed + source state; +- hash spelling, length, encoding, algorithm, and security properties are not + public contracts; +- prompt bodies, templates, source paths, schemas, rendered messages, and + execution settings are not returned; +- no profile, credential, artifact, rendering, schema, validation, capacity, + provider, or model-generation work is performed; +- the returned result and input slice are caller-owned; - filesystem-backed inspection is a point-in-time lookup and does not freeze - a profile for a later execution; + a definition for later execution; - a nil engine matches `ErrInvalidConfig`; -- a blank ID matches `ErrInvalidRequest`; -- an absent exact ID matches `ErrProfileNotFound` and not `ErrProfileLoad`; -- malformed or unreadable profile data, an unknown backend, or an invalid - resolved target matches `ErrProfileLoad`; -- cancellation observed during profile loading matches `ErrProfileLoad` while +- a blank prompt ID matches `ErrInvalidRequest`; +- an absent exact ID or version matches `ErrPromptNotFound` and not + `ErrPromptLoad`; +- malformed, unreadable, duplicate, ambiguous, referenced-content, or hashing + failures match `ErrPromptLoad`; +- cancellation observed during lookup matches `ErrPromptLoad` while preserving the context error through `errors.Is`; and - the method returns no partial result on error. -Update the `Engine` type GoDoc to include `InspectProfile` among the operations -safe for concurrent calls. Do not imply that injected collaborators become -safe when their existing contracts do not provide that guarantee. +Update the `Engine` type GoDoc to include `InspectPrompt` among operations safe +for concurrent calls. Do not imply that mutable injected filesystems or other +collaborators become safe when their own contracts do not provide that +guarantee. Update `doc.go` in the same stage: -- include `Engine.InspectProfile` in the package's operation list; -- include inspection in the concurrency and caller-ownership summary; and -- list `ProfileInspection` among construction and inspection values without a - stable JSON representation. +- include `Engine.InspectPrompt` in the package operation list; +- include prompt inspection in the concurrency and caller-ownership summary; +- list `PromptInputDefinition` and `PromptInspection` among construction and + inspection values without stable JSON representations; and +- retain the existing statement that exposed hashes are opaque. -Do not add `ProfileInspection` to the stable JSON list. +Do not add either new value to the stable JSON list. ### Internal Result Add this internal value to `internal/domain/domain.go` near -`ExecutionProfile` and `ExecutionTarget`: +`PromptDefinition` and `PromptInput`: ```go -// ProfileInspection is the resolved result of exact profile inspection. -type ProfileInspection struct { - ProfileID string - EffectiveModelParams ExecutionTarget - APIKeyRequired bool +// PromptInspection is the resolved result of exact prompt inspection. +type PromptInspection struct { + PromptID string + PromptVersion string + PromptHash string + DefaultProfileID string + Inputs []PromptInput + OutputContract OutputContract } ``` Do not add JSON or YAML tags. The internal value is a use-case result, not a -file format or persistence contract. +source format or persistence contract. -Add a root conversion in `convert.go`: +The use-case operation must copy the definition's `Inputs` slice before +putting it on this result. `PromptInput` currently contains only scalar fields, +so a new slice with value copies is sufficient. -```go -func fromDomainProfileInspection( - inspection *domain.ProfileInspection, -) *ProfileInspection -``` +### Shared Prompt Selection And Hashing -The conversion must: - -- return `nil` for a nil internal value; -- copy the scalar fields; -- convert the target through the existing `fromDomainExecutionTarget`; and -- therefore use the existing recursive `copyAnyMap` boundary for nested extra - parameters. - -Do not add `APIKeyRequired` to the public `ExecutionTarget`. Keeping the -requirement on `ProfileInspection` preserves the existing stable JSON and -general execution-target contract. - -### Shared Profile Selection - -Add `internal/usecase/profile_inspection.go`. Define one private selection +Add `internal/usecase/prompt_inspection.go`. Define one private selection value: ```go -type resolvedProfileSelection struct { - id string - profile *domain.ExecutionProfile - backend *domain.Backend +type resolvedPromptDefinition struct { + definition *domain.PromptDefinition + hash string } ``` Add a private runner helper: ```go -func (r *Runner) resolveProfileSelection( +func (r *Runner) resolvePromptDefinition( ctx context.Context, - profileID string, -) (*resolvedProfileSelection, error) + promptID string, + promptVersion string, +) (*resolvedPromptDefinition, error) ``` The helper must perform these operations in order: -1. trim surrounding whitespace from `profileID`; -2. reject a blank result with `ErrInvalidRequest`; -3. reject a nil runner profile repository with `ErrProfileLoad` rather than - panicking; -4. call the existing `profile.Repository.GetProfile` exactly once; -5. wrap every repository error with `ErrProfileLoad` while preserving the - underlying error with `%w`; -6. reject a nil profile returned without an error as `ErrProfileLoad`; -7. make a value copy of the selected profile before normalization so the - runner does not mutate repository-owned state; -8. trim the copied profile's backend ID; -9. when that ID is nonblank, resolve it exactly once through the existing - `BackendResolver`, preserving the current unknown-backend - `ErrProfileLoad` wrapping and backend ID context; and -10. return the normalized exact ID, copied profile, and optional defensive - backend value. +1. reject a blank or whitespace-only `promptID` with `ErrInvalidRequest`; +2. pass every nonblank ID and the version to the repository unchanged; +3. reject a nil runner or nil prompt repository with `ErrPromptLoad` rather + than panicking; +4. call `promptdef.Repository.GetPromptDefinition` exactly once; +5. wrap every repository error as `ErrPromptLoad` with `%w` while preserving + the repository error, including `promptdef.ErrPromptDefinitionNotFound`; +6. reject a nil definition returned without an error as `ErrPromptLoad`; +7. calculate the equality value with the existing + `hashPromptDefinition` function exactly once; +8. classify a hash failure as `ErrPromptLoad` using the existing preparation + wording and behavior; and +9. return the repository-owned definition for internal per-call use and the + hash. -Do not inspect all sources, enumerate profiles, expose a source path, fall back -after a malformed higher-precedence match, or infer a backend from an endpoint -or model. +Do not trim a nonblank ID or version, clone the complete prompt definition, +parse its templates, inspect multiple definitions, load schemas, or resolve +the default profile. -Refactor the profile-loading and backend-resolution block in -`Runner.resolvePreparation` to call `resolveProfileSelection`. Leave prompt -selection, prompt default-profile selection, prompt hashing, target overrides, -request credentials, output-contract resolution, artifact work, and all later -ordering where they currently occur. +Refactor the prompt-loading and hashing block in +`Runner.resolvePreparation` to call `resolvePromptDefinition`. Retain the +existing early blank-ID check before direct session normalization so a request +with both a blank prompt ID and an invalid direct session preserves its +current error priority. The shared helper may defensively repeat the blank-ID +check. + +After the helper returns, ordinary preparation uses +`selection.definition` and `selection.hash` exactly where it currently uses +the repository result and prompt hash. Leave direct-session normalization, +profile selection, target resolution, credentials, output-contract +resolution, artifacts, schemas, rendering, admission, generation, and all +later ordering unchanged. This refactor must preserve: -- explicit request profile selection over prompt `default_profile`; -- the existing `ErrProfileRequired` result when neither exists; -- exact repository and backend lookup behavior; -- existing public not-found versus profile-load classification; -- request override precedence and target-presence tracking; -- credential checking during ordinary preparation; and -- current `Prepare`, `PrepareExecution`, `Run`, and `RunPrepared` behavior. +- current prompt ID and version selection behavior; +- existing public not-found versus prompt-load classification; +- prompt default-profile selection; +- prompt hashing before any request-specific definition copy or session + template clearing; +- `PreparedRun.PromptHash`, `RunResult.PromptHash`, and prepared-execution + behavior; and +- all current `Prepare`, `PrepareExecution`, `Run`, and `RunPrepared` results + and failure ordering. -### Shared Target Resolution And Structural Validation +Do not change `hashPromptDefinition` or introduce a second equality +calculation. Its current serialization and SHA-256 implementation remain an +internal mechanism behind the opaque public value. -Continue using the existing `resolveExecutionTarget` for framework, backend, -profile, and optional request precedence. Do not duplicate or move the -individual merge rules into the inspection code. +### Internal Inspection Operation -Extract the two existing effective-target requiredness checks from -`resolvePreparation` into a private pure helper: +Add this method in `internal/usecase/prompt_inspection.go`: ```go -func validateResolvedExecutionTarget( - target domain.ExecutionTarget, -) error -``` - -It checks only that the trimmed endpoint and model are nonblank and returns a -plain descriptive error. It does not validate URL syntax, contact the -endpoint, validate credentials, or introduce new parameter rules. - -`resolvePreparation` calls this helper at the same current point: after -request override resolution and direct API-key assignment, and before -`validateAPIKey`. It wraps a failure with `ErrInvalidRequest`, preserving -ordinary preparation behavior. - -Add the internal inspection method: - -```go -func (r *Runner) InspectProfile( +func (r *Runner) InspectPrompt( ctx context.Context, - profileID string, -) (*domain.ProfileInspection, error) + promptID string, + promptVersion string, +) (*domain.PromptInspection, error) ``` It performs these operations: -1. trim and reject a blank ID as `ErrInvalidRequest`; +1. reject a blank or whitespace-only ID as `ErrInvalidRequest`; 2. if the supplied context is already canceled, return an error wrapping both - `ErrProfileLoad` and `ctx.Err()` without touching the repository; -3. call `resolveProfileSelection`; -4. call `resolveExecutionTarget` with the selected backend, selected profile, - and a nil request override; -5. wrap any target-resolution error with `ErrProfileLoad`; -6. call `validateResolvedExecutionTarget` and wrap a failure with - `ErrProfileLoad`; -7. defensively clear `target.APIKey`; -8. copy `target.APIKeyRequired` into the result's `APIKeyRequired`; and -9. return the normalized profile ID and effective target. + `ErrPromptLoad` and `ctx.Err()` without consulting the repository; +3. call `resolvePromptDefinition` with the original ID and version; +4. allocate and copy the selected definition's input slice in its existing + order; +5. return the selected definition's normalized ID, version, default profile, + declared `Validation` contract, copied inputs, and shared prompt hash. -Do not call `validateAPIKey`, `os.Getenv`, prompt repositories, artifact -readers, renderers, validators, capacity admission, output repair, or the -model client. Do not populate request target-presence state. +The returned `OutputContract` comes directly from the loaded definition's +normalized `Validation` value. Do not call `resolveOutputContract`, because no +request override participates in inspection. -The internal `ExecutionTarget` may retain its private `APIKeyRequired` field; -the root conversion intentionally omits that private field from the public -target and publishes the separate inspection boolean. +Do not call profile or backend repositories, `resolveProfileSelection`, +`resolveExecutionTarget`, credential checks, schema loaders, artifact readers, +renderers, validators, capacity admission, output repair, or the model client. +Do not return the definition itself. + +The operation returns no partial result. Repository cancellation that occurs +after lookup begins remains wrapped by `ErrPromptLoad` through the shared +helper while preserving the underlying context error where the repository +does so. + +### Root Conversion + +Add this conversion in `convert.go` near +`fromDomainProfileInspection`: + +```go +func fromDomainPromptInspection( + inspection *domain.PromptInspection, +) *PromptInspection +``` + +The conversion must: + +- return `nil` for a nil internal value; +- copy every scalar identity, hash, and default-profile field; +- allocate a new `[]PromptInputDefinition` in the same order and copy every + name, required flag, content type, and description; +- convert the contract through the existing + `fromDomainOutputContract`; and +- return a non-aliased root value. + +Do not expose internal `PromptInput` or `PromptDefinition` types at the root +boundary. Do not use serialization as a copying mechanism. ### Root Facade And Error Mapping -`Engine.InspectProfile` follows the existing facade pattern: +`Engine.InspectPrompt` follows the existing inspection facade pattern: 1. reject a nil engine or nil runner with `ErrInvalidConfig`; -2. pass the context and string ID directly to `Runner.InspectProfile`; +2. pass the context, ID, and version directly to + `Runner.InspectPrompt`; 3. map internal failures through the existing `mapPublicError`; and -4. convert a successful result with `fromDomainProfileInspection`. +4. convert a successful result with `fromDomainPromptInspection`. No request conversion is needed. Do not add a public sentinel or typed error. The existing error mapping already has the required ordering: -- an underlying `profile.ErrProfileNotFound` maps to - `ErrProfileNotFound` before the enclosing use-case `ErrProfileLoad` is +- an underlying `promptdef.ErrPromptDefinitionNotFound` maps to + `ErrPromptNotFound` before the enclosing use-case `ErrPromptLoad` is considered; -- other use-case `ErrProfileLoad` failures map to `ErrProfileLoad`; and +- other `usecase.ErrPromptLoad` failures map to `ErrPromptLoad`; and - `usecase.ErrInvalidRequest` maps to `ErrInvalidRequest`. -Do not reorder or otherwise change `errors.go` unless a focused test proves -that the existing mapping does not meet this plan. Preserve underlying -repository, backend, and context errors through `errors.Is`. +Do not reorder or otherwise change `errors.go` unless a focused public test +proves the current mapping fails a required identity. Preserve underlying +repository and context errors through `errors.Is`. -### Credentials, Ownership, And Concurrency +### Ownership, Consistency, And Concurrency -Inspection has no input through which a direct credential value can enter. -Backend and file-profile `APIKeyEnv` values remain names only. An in-memory -profile with `APIKeyRequired` clears an inherited backend environment name -through the existing target merge behavior. +The internal result has a copied input slice, and the root conversion creates +another public slice. A consumer may mutate the result and its inputs without +affecting: -The implementation must succeed for: - -- a backend or profile naming an unset environment variable; -- a backend or profile naming an environment variable whose value is blank; -- an in-memory profile requiring a direct key when no key is supplied; and -- a profile requiring no credential. - -The result expresses these effective states: - -| `EffectiveModelParams.APIKeyEnv` | `APIKeyRequired` | Meaning | -| --- | --- | --- | -| nonblank | `false` | The profile resolves to the named environment source. | -| blank | `true` | A later execution must supply a direct key or explicit request environment override. | -| blank | `false` | The resolved target declares no credential requirement. | - -The implementation must never produce a successful public result with both a -nonblank `APIKeyEnv` and `APIKeyRequired == true`. - -Public ownership is enforced at the root conversion boundary. A consumer may -mutate the returned target and arbitrarily nested JSON-compatible extra -parameters without affecting: - -- the engine registry or profile repository; -- a later `InspectProfile` call; +- the prompt repository or selected definition; +- the engine; +- a later `InspectPrompt` call; - `Prepare`, prepared execution, or `Run`; or - another result already returned to a caller. -No new mutable engine state, cache, global registry, lock, or goroutine is -needed. Concurrency safety follows from the existing immutable registry and -repository contracts plus per-call values. +`OutputContract` and each input element contain scalar values, so no deeper +mutable tree exists in the accepted result shape. + +No new mutable engine state, source cache, global registry, lock, or goroutine +is needed. Concurrency safety follows from existing repository contracts and +per-call result allocation. Inspection does not freeze mutable source state; +prepared execution remains the exact snapshot-to-execution workflow. ### Test Ownership Add focused internal tests in -`internal/usecase/profile_inspection_test.go`. Use small repository and backend -fakes already present in the package where practical; do not build a parallel -fixture framework. +`internal/usecase/prompt_inspection_test.go`. Reuse package fakes where doing +so remains clearer than adding a new fixture, or define one small counting +prompt repository local to this test file. The internal tests own: -- blank-ID rejection; -- pre-canceled context classification without repository access; -- one exact profile and backend lookup; -- framework-default, backend, and profile target precedence through the - existing resolver; -- endpoint-only behavior; -- effective environment-name reporting without availability checks; -- direct-key-required behavior clearing an inherited backend environment - name; -- missing-profile and unknown-backend wrapping; and -- nil repository, nil returned profile, and invalid resolved target defenses - only if these cases are not already cheaply covered through existing runner - fakes. +- blank-ID rejection without repository access; +- a pre-canceled context matching both `ErrPromptLoad` and the context error + without repository access; +- exactly one repository lookup with the original nonblank ID and version; +- a successful result containing normalized identity, hash, default profile, + complete ordered input metadata, and the declared output contract; +- successful operation with every non-prompt runner collaborator nil; +- copied inputs across repeated inspections; +- missing-prompt wrapping that retains + `promptdef.ErrPromptDefinitionNotFound`; +- nil repository and nil returned definition defenses; and +- representative preservation of ordinary preparation after it is switched + to the shared selection-and-hash helper. -Keep the internal matrix compact. Profile parser tests continue to own YAML, -duplicates, raw-key rejection, ranges, and source discovery. Backend tests -continue to own registration validation. Existing target tests continue to -own every merge field and numeric override boundary. +Combine these into a small number of readable behavioral tests. Prompt +repository tests remain the owners of YAML discovery, strict decoding, +normalization, content-file containment and reading, duplicate selection, and +ID/version matrices. Existing preparation tests remain the owners of session, +profile, artifact, schema, rendering, credential, and execution ordering. Add external-package public contract tests in `public_contract_test.go`. They own: - the exported method and result shape through normal Go use; -- a nil engine returning `ErrInvalidConfig`; -- blank, missing, malformed, and unknown-backend public error identities, - including that not-found does not also match `ErrProfileLoad`; -- a pre-canceled inspection preserving both `ErrProfileLoad` and the context - error without consulting the profile repository; -- successful inspection with an absent credential environment value; -- no prompt dependency, demonstrated with an empty configured prompt - `fs.FS`; -- no model invocation, using the existing deterministic fake client; -- the three credential-requirement result states; -- an endpoint-only profile's empty backend ID; -- deep caller ownership of nested extra parameters across repeated - inspections; and -- representative equivalence between `InspectProfile.EffectiveModelParams` - and `Prepare.EffectiveModelParams` for the same engine, profile, and source - state with no request execution override. +- successful exact-version inspection from an `fs.FS` prompt source with + multiple versions; +- normalized input and output metadata, including a declared JSON Schema path, + without configuring or loading a schema source; +- referenced `content_file` loading without returning its body; +- no default-profile resolution, demonstrated by a nonexistent declared + profile; +- no model invocation, using an existing deterministic fake client where + useful; +- nil-engine, blank-ID, missing exact version, ambiguous omitted version, + malformed definition or referenced-content, and pre-canceled public error + identities; +- that not-found does not also match `ErrPromptLoad`; +- caller ownership across repeated inspection and later preparation; and +- equality between `PromptInspection.PromptHash` and + `PreparedRun.PromptHash` for the same prompt and observed source state. -Combine closely related assertions into a few readable behavioral tests. -Do not add a JSON golden test because `ProfileInspection` deliberately has no -stable JSON contract. Do not duplicate the complete repository-precedence, -target-field, parser-error, or credential-execution suites at the root layer. +Use a simple executable prompt with an in-memory profile for the hash +equivalence test. Use a separate JSON Schema declaration for the +schema-independent inspection test so `Prepare` is not needed there. -Existing tests that must continue passing without semantic edits include: +Do not add JSON golden or round-trip tests because the new inspection values +deliberately have no stable JSON contract. Do not duplicate the full +prompt-repository parser, source-option precedence, schema-validation, +preparation, or public error suites at the root layer. -- profile repository and built-in fallback tests; -- backend registry defensive-copy and lookup tests; -- execution-target precedence tests; -- `Prepare` and `Run` profile/backend equivalence tests; -- prepared-execution snapshot and credential tests; and -- public error mapping tests. +Existing tests that must continue passing without weakened assertions include: + +- prompt-definition filesystem and `fs.FS` repository tests; +- prompt-source option replacement tests; +- `Prepare` and `Run` prompt selection and hashing tests; +- direct-session prompt-hash invariance tests; +- prepared-execution frozen-source and details tests; +- public error mapping tests; and +- profile inspection tests. ### Documentation Ownership After the implementation and public tests pass, update current-state documentation: -- `docs/consumers/pkg-promptkit.md`: add a concise task-oriented section near - profile selection showing `Engine.InspectProfile`, explaining when to use it - instead of a synthetic `Prepare`, how to interpret `APIKeyEnv` and - `APIKeyRequired`, and that credential enforcement timing remains - application policy. Link to the exact GoDoc rather than restating every - error and field contract. -- `docs/formats.md`: update profile selection and backend-membership wording - so exact inspection is recognized as another consumer of the existing - source and profile precedence. Do not redefine the exported method here. -- `docs/internal/runner.md`: describe the shared profile-selection boundary - and explain that inspection stops after structural target resolution, - before credential availability and all prompt-dependent work. -- `docs/internal/sources.md`: record that exact profile inspection performs - one point-in-time profile-source lookup without reading prompt, input, or - schema sources, and replace any ambiguous use of “inspection value” for - `PreparedRun` with “preparation value.” -- `docs/internal/overview.md`: add profile inspection to the existing root +- `docs/consumers/pkg-promptkit.md`: add a concise task-oriented section after + engine construction showing `Engine.InspectPrompt`, explaining how a + consumer can validate declared inputs and output workflow before preparing, + and distinguishing it from `InspectProfile`, `Prepare`, and prepared + execution. Link to GoDoc for exact fields and errors. +- `docs/formats.md`: note that exact prompt inspection uses the same prompt + source, strict decoding, content-file resolution, and ID/version selection + described by the format reference. Do not redefine the exported method. +- `docs/internal/runner.md`: add a shared prompt-selection boundary for + inspection and preparation, including shared hashing, and explain that + inspection stops before every execution-dependent collaborator. +- `docs/internal/sources.md`: record that exact prompt inspection performs one + point-in-time prompt lookup, validates referenced message content through + the repository, and does not parse templates or read profile, input, or + schema sources. +- `docs/internal/overview.md`: add prompt inspection to the existing root facade and `internal/usecase` responsibility descriptions. Do not add a new component or package row. -- `docs/roadmap/future.md`: remove the statement that profile inspection is in - active planning and leave the remaining unselected ideas intact. -- `docs/roadmap/notarius-promptkit-wishlist.md` and - `docs/roadmap/weatherreporter-promptkit-wishlist.md`: change the profile - inspection disposition from accepted planning to implemented behavior and - link to the durable consumer guidance or GoDoc rather than treating the - roadmap as current-state documentation. -- `docs/roadmap/profile-inspection.md`: change its status to `Complete` only - after the code, tests, current-state documentation, and full validation are +- `docs/roadmap/future.md`: remove the statement that prompt inspection is in + active planning and leave structured capacity errors intact. +- `docs/roadmap/weatherreporter-promptkit-wishlist.md`: change the prompt + inspection disposition from accepted planning to implemented behavior, + link to durable consumer guidance or GoDoc, and remove or condense proposed + API detail that would compete with the implemented declarations. +- `docs/roadmap/prompt-inspection.md`: change its status to `Complete` only + after code, tests, current-state documentation, and full validation are complete. - `docs/roadmap/implementation.md`: change its status to `Complete` only after every completion gate in this plan is satisfied. -Do not update release guidance in this feature implementation. A later release -pass decides whether the change warrants a supplemental release document. +The accepted feature roadmap already describes only purpose, scope, policy, +and target end state; it contains no stage sequence to remove during this +planning pass. -## Stage 1: Implement Shared Internal Profile Resolution +Do not update release guidance in this feature implementation. A later release +pass decides whether the additive API warrants a supplemental release +document. + +## Stage 1: Share Prompt Selection And Implement Internal Inspection ### Objective -Add the internal inspection result and operation, share profile/backend -selection and structural target validation with ordinary preparation, and -prove the internal behavior without publishing the root API yet. +Add the internal inspection result and operation, share exact prompt loading +and hashing with ordinary preparation, and prove the internal behavior without +publishing the root API yet. ### Implementation Prompt Implement only Stage 1 of -`docs/roadmap/implementation.md`. Read the complete feature roadmap and the -implementation rules and fixed design above before editing. +`docs/roadmap/implementation.md`. Read the complete feature roadmap, +implementation rules, and fixed design above before editing. -1. Add `domain.ProfileInspection` to `internal/domain/domain.go` without - serialization tags. -2. Add `internal/usecase/profile_inspection.go` with - `resolvedProfileSelection`, `resolveProfileSelection`, - `validateResolvedExecutionTarget`, and `Runner.InspectProfile` exactly as - specified. +1. Add `domain.PromptInspection` to `internal/domain/domain.go` without source + or serialization tags. +2. Add `internal/usecase/prompt_inspection.go` with + `resolvedPromptDefinition`, `resolvePromptDefinition`, and + `Runner.InspectPrompt` exactly as specified. 3. Refactor `Runner.resolvePreparation` in `internal/usecase/runner.go` to use - the shared selection and target-validation helpers while preserving its - current ordering, error classification, target overrides, credentials, and - results. + the shared prompt selection and hash while preserving existing validation + and execution ordering. 4. Add lean behavioral tests in - `internal/usecase/profile_inspection_test.go`. + `internal/usecase/prompt_inspection_test.go`. 5. Run the focused validation below and repair regressions before ending the stage. Do not add the root public type or method, update current-state documentation, -or alter profile formats, backend registration, credential availability, -capacity, model-client, or provider behavior in this stage. +or alter prompt formats, template behavior, profile selection, schema +handling, capacity, validation, model-client, or provider behavior in this +stage. ### Focused Validation @@ -487,28 +507,31 @@ Run: ```sh gofmt -w internal/domain/domain.go \ - internal/usecase/profile_inspection.go \ - internal/usecase/profile_inspection_test.go \ + internal/usecase/prompt_inspection.go \ + internal/usecase/prompt_inspection_test.go \ internal/usecase/runner.go -go test ./internal/usecase ./internal/profile ./internal/profile/builtin \ - ./internal/backend +go test ./internal/usecase ./internal/promptdef go test ./internal/usecase -run \ - 'TestRunner(InspectProfile|Prepare|Run|PrepareExecution|RunPrepared)' -go vet ./internal/usecase ./internal/profile ./internal/backend + 'TestRunner(InspectPrompt|Prepare|Run|PrepareExecution|RunPrepared)' +go vet ./internal/usecase ./internal/promptdef ``` +If actual existing test names do not match the focused expression, run the +smallest truthful package or expression that covers the listed behavior +rather than weakening or skipping assertions. + ### Completion Gate Stage 1 is complete only when: -- inspection reaches profile and backend resolution without any prompt or - execution collaborator; -- profile/backend selection is shared with ordinary preparation; -- target merging and requiredness checks are shared rather than duplicated; -- inspection does not check credential values or capacity; -- internal errors preserve the required identities; -- the ordinary preparation and execution tests still pass without weakened - assertions; and +- inspection loads and hashes one exact prompt through the configured + repository; +- prompt selection and hashing are shared with ordinary preparation; +- nonblank ID and version values reach the repository unchanged; +- successful inspection needs no non-prompt runner collaborator; +- inputs are copied and no definition or content body escapes; +- cancellation and load failures preserve the required internal identities; +- ordinary preparation and execution behavior remains unchanged; and - no root public API or current-state documentation claims the feature yet. ## Stage 2: Publish The Root Facade And Public Contract @@ -523,30 +546,29 @@ contract. Implement only Stage 2 of `docs/roadmap/implementation.md` after Stage 1 satisfies its completion gate. -Re-read the fixed public API, error, credential, ownership, and test sections +Re-read the fixed public API, conversion, error, ownership, and test sections before editing. -1. Add `ProfileInspection` and its exact GoDoc to `types.go` immediately after - `ExecutionTarget`. Do not add JSON tags or change `ExecutionTarget`. -2. Add `fromDomainProfileInspection` to `convert.go` using - `fromDomainExecutionTarget`. -3. Add `Engine.InspectProfile` and its exact GoDoc to `engine.go` immediately - before `Prepare`. -4. Update `Engine` GoDoc to include concurrent inspection. -5. Update the package GoDoc in `doc.go` for operation discovery, concurrency, - ownership, and the non-stable JSON classification. +1. Add `PromptInputDefinition` and `PromptInspection` with exact GoDoc to + `types.go` immediately after `ProfileInspection`. Do not add JSON tags. +2. Add `fromDomainPromptInspection` to `convert.go` using a newly allocated + input slice and the existing output-contract converter. +3. Add `Engine.InspectPrompt` and exact GoDoc to `engine.go` immediately before + `InspectProfile`. +4. Update `Engine` GoDoc to include concurrent prompt inspection. +5. Update package GoDoc in `doc.go` for operation discovery, concurrency, + ownership, opaque hashes, and non-stable JSON classification. 6. Add compact external-package contract coverage in - `public_contract_test.go`, using existing fixtures and fakes where they + `public_contract_test.go`, reusing existing fixtures and fakes where they remain clear. -7. Confirm that the existing `errors.go` mapping meets the plan; do not change - it unless a required public identity test fails for a genuine mapping - reason. +7. Confirm that existing `errors.go` mapping meets the plan; do not change it + unless a required public identity test fails for a genuine mapping reason. 8. Run the focused validation below and repair regressions before ending the stage. -Do not add enumeration, request overrides, profile fingerprints, JSON -stability, prompt inspection, environment credential checks, caching, or -current-state prose documentation in this stage. +Do not add enumeration, source validation across a corpus, request overrides, +template or schema output, profile resolution, JSON stability, caching, new +errors, or current-state prose documentation in this stage. ### Focused Validation @@ -555,8 +577,9 @@ Run: ```sh gofmt -w doc.go types.go convert.go engine.go public_contract_test.go go test . -go test ./internal/usecase -go test . -run 'Test(InspectProfile|.*Profile.*PublicError|.*Profile.*Contract)' +go test ./internal/usecase ./internal/promptdef +go test . -run \ + 'Test(InspectPrompt|.*Prompt.*PublicError|.*Prompt.*Contract)' go vet . go build . ``` @@ -568,25 +591,26 @@ names rather than weakening or skipping the intended assertions. Stage 2 is complete only when: -- a consumer can call `Engine.InspectProfile` through the root package; -- the result contains only the normalized ID, caller-owned effective target, - and direct-key-required signal; -- no credential value can enter the result; -- environment, direct, and no-credential states are unambiguous; -- nil, blank, missing, malformed, unknown-backend, and cancellation errors - have the required public identities; -- not-found remains distinct from profile-load failure; -- the effective target matches ordinary preparation absent request overrides; +- a consumer can call `Engine.InspectPrompt` through the root package; +- exact ID/version selection matches ordinary preparation; +- the result contains only accepted identity, input, default-profile, + output-contract, and opaque equality metadata; +- no prompt body, schema body, profile, credential, or execution setting is + exposed or resolved; +- nil, blank, missing, malformed, ambiguous, referenced-content, and + cancellation errors have the required public identities; +- not-found remains distinct from prompt-load failure; +- `PromptHash` matches ordinary preparation for the same source state; - repeated calls and caller mutation cannot alter engine-owned state; - no stable JSON or new error contract was introduced; and -- existing public preparation, execution, and stable JSON tests remain - unchanged and passing. +- existing public preparation, execution, profile-inspection, and stable JSON + tests remain unchanged and passing. ## Stage 3: Update Documentation And Validate The Repository ### Objective -Make implemented profile inspection discoverable in its canonical +Make implemented prompt inspection discoverable in its canonical documentation, reconcile temporary roadmap state, and complete full repository validation. @@ -600,15 +624,15 @@ gates. `docs/internal/runner.md`, `docs/internal/sources.md`, and `docs/internal/overview.md` according to the documentation ownership section above. -2. Update the future catalog and both downstream wishlist dispositions so they - no longer describe profile inspection as merely accepted work. +2. Update the future catalog and Weatherreporter wishlist disposition so they + no longer describe prompt inspection as merely accepted work. 3. Check every changed Markdown link and confirm its file and heading target. 4. Run the full validation sequence below. 5. Only after every check passes, set the feature roadmap and this implementation plan to `**Status:** Complete.` 6. Re-run `git diff --check` after the status edits. -Do not add release notes, an example program, a new public package, or a +Do not add release notes, a new example program, a new public package, or a duplicate API reference. Keep detailed contracts in GoDoc and task-oriented usage in the consumer guide. @@ -618,8 +642,8 @@ Run from the repository root: ```sh gofmt -w internal/domain/domain.go \ - internal/usecase/profile_inspection.go \ - internal/usecase/profile_inspection_test.go \ + internal/usecase/prompt_inspection.go \ + internal/usecase/prompt_inspection_test.go \ internal/usecase/runner.go \ doc.go types.go convert.go engine.go public_contract_test.go gofmt -l $(git ls-files '*.go') @@ -639,13 +663,13 @@ Inspect the final diff and confirm: - only files required by this feature and pre-existing user changes are present; -- no credential values, private infrastructure details, workspace files, - local replacements, generated binaries, or unrelated formatting changes - were added; +- no credentials, private source content, workspace files, local + replacements, generated binaries, or unrelated formatting changes were + added; - the public declarations and GoDoc own exact API behavior; - current-state documentation describes only implemented behavior and links to canonical owners; -- roadmap documents contain future scope or completion status rather than a +- roadmap documents contain scope or completion status rather than a duplicate current API reference; and - no release, commit, or tag was created. diff --git a/docs/roadmap/prompt-inspection.md b/docs/roadmap/prompt-inspection.md new file mode 100644 index 0000000..58dc0a8 --- /dev/null +++ b/docs/roadmap/prompt-inspection.md @@ -0,0 +1,268 @@ +# Prompt-Definition Inspection + +**Status:** Accepted. + +## Purpose + +Allow consumers to look up one prompt definition by ID and optional version +and inspect its declared interface without supplying placeholder inputs, +resolving a profile, rendering templates, loading schemas, or invoking a +model. + +This provides a focused configuration-validation boundary for Weatherreporter, +whose need is recorded in its +[Promptkit wishlist](weatherreporter-promptkit-wishlist.md#priority-2-prompt-definition-inspection), +while preserving Promptkit's deferred, request-oriented source model. + +## Motivation + +Weatherreporter has a fixed application-owned registry of report definitions. +It needs to verify that every configured prompt exists and declares the +expected inputs and output workflow before collecting weather data or starting +provider work. + +The current workaround is to construct synthetic artifacts and variables and +call `Engine.Prepare`. That validates much more than the application needs: +prompt inputs are loaded, templates and session IDs are rendered, a profile +and backend are resolved, credentials are checked, and JSON Schemas may be +loaded and compiled. + +Promptkit already owns prompt source selection, exact ID and version lookup, +strict definition decoding, referenced content-file resolution, and prompt +hashing. It should expose that cohesive subset directly rather than requiring +consumers to reproduce its rules or maintain placeholder execution fixtures. + +## Consumer Workflow + +The target public workflow is: + +```go +inspection, err := engine.InspectPrompt(ctx, promptID, promptVersion) +if err != nil { + // Reject or report the configured prompt. + return +} + +for _, input := range inspection.Inputs { + // Compare the declared prompt interface with application configuration. +} +``` + +The target public surface is: + +```go +type PromptInputDefinition struct { + Name string + Required bool + ContentType string + Description string +} + +type PromptInspection struct { + PromptID string + PromptVersion string + PromptHash string + DefaultProfileID string + Inputs []PromptInputDefinition + OutputContract OutputContract +} + +func (e *Engine) InspectPrompt( + ctx context.Context, + promptID string, + promptVersion string, +) (*PromptInspection, error) +``` + +The exported declarations and GoDoc will own the exact implemented contract. +The important public shape is exact lookup through an existing engine, one +caller-owned inspection value, declared input metadata, the declared output +contract, and the same opaque prompt equality value used by preparation. + +`PromptInspection` and `PromptInputDefinition` do not need stable JSON +representations. Consumers that persist application configuration or +diagnostics can define their own format and select the fields they require. +The existing stable representations of `OutputContract`, `OutputFormat`, and +`ValidationMode` remain unchanged. + +## Lookup And Source Selection + +`InspectPrompt` requires a non-blank prompt ID and performs the same +case-sensitive exact selection used by ordinary preparation. Inspection does +not trim or otherwise canonicalize a non-blank ID or version independently of +that execution path. + +When `promptVersion` is non-empty, the configured prompt source must contain +exactly one matching ID and version pair. When it is empty, the source must +contain exactly one definition with the requested ID; multiple matching +versions remain an ambiguous source-selection error. + +Lookup uses the engine's normally selected prompt source: + +- the last `WithPromptFS` or `WithPromptFile` option replaces earlier prompt + source options and `Config.PromptDir`; or +- `Config.PromptDir` supplies the source when no prompt source option is + present. + +Inspection does not merge prompt sources, fall back after a malformed match, +or introduce a new public prompt repository. The +[format reference](../formats.md#prompt-definitions) remains the canonical +owner of prompt source and exact selection behavior. + +## Structural Validation + +A successful inspection proves that the selected definition can be loaded +through the ordinary prompt repository and satisfies its source-level +structural rules. This includes: + +- deterministic YAML discovery and exact definition selection; +- strict decoding and validation of required identity, version, messages, and + output fields; +- normalization and validation of input declarations, message roles, cache + control, the default-profile identifier, and output-contract fields; and +- contained, readable resolution of every referenced message + `content_file`. + +Referenced message content participates in validation and hashing but is not +returned. Inspection does not parse or execute Go templates, resolve template +variables or input helpers, or enforce the runtime presence and media type of +declared inputs. Those checks remain part of rendering and preparation. + +The returned `OutputContract` is the normalized contract declared by the +prompt definition. For `json_schema` validation, structural inspection proves +that a non-blank schema path is declared and returns that path. It does not +open, resolve, compile, or return the schema document or its references. +Schema-source validation remains part of executable preparation and any +future explicit full-source validation feature. + +Inspection returns `DefaultProfileID` as declared metadata. It does not look +up that profile, resolve a backend or execution target, check credentials, or +prove that the profile is executable. Consumers may call `InspectProfile` +separately when their application policy requires that additional check. + +## Result Semantics + +The result exposes the complete declared input metadata in definition order: +name, required status, content type, and human-readable description. It does +not expose message templates, session-ID templates, cache-control details, +prompt descriptions, raw YAML, source paths, rendered messages, or schema +bodies. + +`PromptID` and `PromptVersion` are the validated identity read from the +selected definition. `PromptHash` is an opaque equality value for the complete +loaded definition, including referenced message content. For the same selected +definition and observed source state, it must equal the `PreparedRun.PromptHash` +produced by ordinary preparation. + +Consumers may compare `PromptHash` values but must not depend on their +algorithm, encoding, length, or suitability as a security proof. Promptkit may +change the representation in a future release together with the existing +prepared-run hash contract. + +## Ownership, Consistency, And Concurrency + +Each successful call returns a caller-owned snapshot. Mutating the result or +its input slice cannot affect the engine, later inspections, or later +preparation. + +Inspection is safe to call concurrently under the engine's existing +repository contracts. It does not mutate prompt sources or install a global +cache. + +For filesystem-backed and mutable `fs.FS` sources, the result describes the +state observed by that lookup. It does not freeze the definition for a later +`Prepare`, `PrepareExecution`, or `Run`, and a source may change between +operations. Consumers requiring an exact preflight-to-execution snapshot +should use the existing prepared-execution workflow. + +## Errors And Cancellation + +The operation uses existing public error categories: + +- a blank or whitespace-only prompt ID matches `ErrInvalidRequest`; +- an absent exact ID or version matches `ErrPromptNotFound` and not + `ErrPromptLoad`; +- read, strict-decode, validation, duplicate, ambiguity, referenced-content, + and prompt-hashing failures match `ErrPromptLoad`; and +- a nil engine receiver matches `ErrInvalidConfig`. + +Errors preserve useful underlying collaborator and context identities through +`errors.Is` where the existing facade does so, without exposing internal +package types. Context cancellation governs the complete lookup, and no +partial inspection result is returned on failure. + +Inspection cannot return profile, credential, artifact, rendering, schema, +capacity, validation, or model-generation errors because it does not perform +those operations. + +## Compatibility And Boundaries + +This feature is additive. Existing prompt formats, prompt source selection, +`Prepare`, prepared execution, `Run`, profile inspection, and stable JSON +contracts remain unchanged. + +The method belongs on the root `Engine` facade. Prompt repositories and +internal domain definitions remain implementation details, and consumers +cannot use inspection to replace or mutate registered prompt definitions. +Engine construction retains its ordinary configuration requirements; this +feature does not introduce a separate prompt-only engine. + +Inspection is intentionally definition-oriented rather than +execution-oriented. It reports what the prompt declares, not an effective +request after profile selection, per-run overrides, artifacts, variables, +schema resolution, or rendering. + +## Documentation + +When implemented, documentation should retain these ownership boundaries: + +- exported declarations and GoDoc own the exact method, result, ownership, + hash, error, and cancellation contracts; +- the Promptkit consumer guide explains configuration-time prompt inspection + and distinguishes it from profile inspection, `Prepare`, and prepared + execution; +- the format reference continues to own prompt fields, source selection, + content-file resolution, and version behavior; and +- internal documentation describes shared prompt loading and hashing without + duplicating the public contract. + +## Non-Goals + +This work does not include: + +- enumerating, searching, or filtering prompt definitions; +- validating every definition across one or more prompt sources; +- returning raw definitions, YAML, source paths, message bodies, rendered + messages, prompt descriptions, or session-ID templates; +- parsing or rendering templates or accepting placeholder inputs and + variables; +- resolving the default profile, backend, effective execution target, or + credentials; +- loading, compiling, or returning JSON Schema documents or transitive + references; +- validating application-specific relationships among prompts; +- freezing a mutable prompt source for later execution; +- exposing or mutating an internal prompt repository; +- adding a stable JSON representation for the new inspection values; or +- changing current prompt selection, hashing, preparation, or execution + behavior. + +## Target End State + +After this work: + +- consumers can validate one configured prompt without inventing artifacts, + variables, or an executable profile; +- lookup uses ordinary prompt-source selection and exact ID/version semantics; +- a successful result proves that the selected definition and its referenced + message content are structurally loadable; +- callers receive validated identity, complete declared input metadata, the + default profile ID, the normalized output contract, and an opaque prompt + equality value; +- the equality value matches ordinary preparation for the same selected + definition and observed source state; +- no prompt body, rendered message, schema body, credential, or execution + target is exposed; +- returned values are caller-owned and safe to inspect concurrently; and +- broader corpus validation, enumeration, schema validation, and executable + preparation remain separate responsibilities. diff --git a/docs/roadmap/weatherreporter-promptkit-wishlist.md b/docs/roadmap/weatherreporter-promptkit-wishlist.md index f7c4cf9..b24d9ec 100644 --- a/docs/roadmap/weatherreporter-promptkit-wishlist.md +++ b/docs/roadmap/weatherreporter-promptkit-wishlist.md @@ -107,7 +107,7 @@ describes the actual execution. ## Priority 2: Prompt-Definition Inspection **Disposition:** Accepted into the -[future catalog](future.md#prompt-definition-inspection). +[prompt-definition inspection](prompt-inspection.md) feature roadmap. ### Downstream need diff --git a/internal/domain/domain.go b/internal/domain/domain.go index d4ef634..6c11e14 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -145,6 +145,16 @@ type PromptDefinition struct { Validation OutputContract `yaml:"validation"` } +// PromptInspection is the resolved result of exact prompt inspection. +type PromptInspection struct { + PromptID string + PromptVersion string + PromptHash string + DefaultProfileID string + Inputs []PromptInput + OutputContract OutputContract +} + // PromptInput describes one named input expected by a prompt definition. type PromptInput struct { Name string `yaml:"name"` diff --git a/internal/usecase/prompt_inspection.go b/internal/usecase/prompt_inspection.go new file mode 100644 index 0000000..31f2e2f --- /dev/null +++ b/internal/usecase/prompt_inspection.go @@ -0,0 +1,73 @@ +package usecase + +import ( + "context" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +type resolvedPromptDefinition struct { + definition *domain.PromptDefinition + hash string +} + +func (r *Runner) resolvePromptDefinition( + ctx context.Context, + promptID string, + promptVersion string, +) (*resolvedPromptDefinition, error) { + if strings.TrimSpace(promptID) == "" { + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) + } + if r == nil || r.promptDefs == nil { + return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad) + } + + definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) + } + if definition == nil { + return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad) + } + + hash, err := hashPromptDefinition(definition) + if err != nil { + return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err) + } + return &resolvedPromptDefinition{definition: definition, hash: hash}, nil +} + +// InspectPrompt resolves one explicit prompt without execution work. +func (r *Runner) InspectPrompt( + ctx context.Context, + promptID string, + promptVersion string, +) (*domain.PromptInspection, error) { + if strings.TrimSpace(promptID) == "" { + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err()) + default: + } + + selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion) + if err != nil { + return nil, err + } + inputs := make([]domain.PromptInput, len(selection.definition.Inputs)) + copy(inputs, selection.definition.Inputs) + + return &domain.PromptInspection{ + PromptID: selection.definition.ID, + PromptVersion: selection.definition.Version, + PromptHash: selection.hash, + DefaultProfileID: selection.definition.DefaultProfile, + Inputs: inputs, + OutputContract: selection.definition.Validation, + }, nil +} diff --git a/internal/usecase/prompt_inspection_test.go b/internal/usecase/prompt_inspection_test.go new file mode 100644 index 0000000..9560dd0 --- /dev/null +++ b/internal/usecase/prompt_inspection_test.go @@ -0,0 +1,157 @@ +package usecase + +import ( + "context" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" +) + +type inspectionPromptRepository struct { + definition *domain.PromptDefinition + err error + calls int + id string + version string +} + +func (r *inspectionPromptRepository) GetPromptDefinition( + _ context.Context, + id string, + version string, +) (*domain.PromptDefinition, error) { + r.calls++ + r.id = id + r.version = version + if r.err != nil { + return nil, r.err + } + return r.definition, nil +} + +func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) { + definition := &domain.PromptDefinition{ + ID: "normalized.prompt", + Version: "1.2.3", + DefaultProfile: "not-resolved", + Inputs: []domain.PromptInput{ + {Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."}, + {Name: "audience", ContentType: "text/plain", Description: "Intended reader."}, + }, + Validation: domain.OutputContract{ + Format: domain.FormatJSON, + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schemas/result.json", + RepairAttempts: 2, + }, + } + repository := &inspectionPromptRepository{definition: definition} + runner := &Runner{promptDefs: repository} + + inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ") + if err != nil { + t.Fatalf("inspect prompt: %v", err) + } + wantHash, err := hashPromptDefinition(definition) + if err != nil { + t.Fatalf("hash prompt definition: %v", err) + } + if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " { + t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version) + } + if inspection.PromptID != definition.ID || + inspection.PromptVersion != definition.Version || + inspection.PromptHash != wantHash || + inspection.DefaultProfileID != definition.DefaultProfile || + !reflect.DeepEqual(inspection.Inputs, definition.Inputs) || + inspection.OutputContract != definition.Validation { + t.Fatalf("inspection=%#v, want definition metadata", inspection) + } + + inspection.Inputs[0].Name = "changed" + second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ") + if err != nil { + t.Fatalf("inspect prompt again: %v", err) + } + if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" { + t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs) + } +} + +func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) { + t.Run("blank ID", func(t *testing.T) { + repository := &inspectionPromptRepository{} + _, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version") + if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 { + t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls) + } + }) + + t.Run("canceled context", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + repository := &inspectionPromptRepository{} + _, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version") + if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 { + t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls) + } + }) + + t.Run("missing prompt", func(t *testing.T) { + repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound} + _, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version") + if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) { + t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err) + } + }) + + t.Run("defensive prompt dependencies", func(t *testing.T) { + var nilRunner *Runner + cases := []struct { + name string + runner *Runner + }{ + {name: "nil runner", runner: nilRunner}, + {name: "nil repository", runner: &Runner{}}, + {name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version") + if !errors.Is(err, ErrPromptLoad) { + t.Fatalf("inspection error=%v, want prompt load", err) + } + }) + } + }) +} + +func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) { + definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0) + repository := &fakePromptRepo{def: definition} + runner := NewRunner( + repository, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + nil, + &fakeArtifactReader{}, + &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}}, + nil, + nil, + nil, + ) + + inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version) + if err != nil { + t.Fatalf("inspect prompt: %v", err) + } + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"}) + if err != nil { + t.Fatalf("prepare prompt: %v", err) + } + if inspection.PromptHash != prepared.PromptHash { + t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash) + } +} diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 72f7941..08d6c03 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -266,14 +266,12 @@ func (r *Runner) resolvePreparation( return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err) } - def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion) + promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion) if err != nil { - return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) - } - promptDefinitionHash, err := hashPromptDefinition(def) - if err != nil { - return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err) + return nil, err } + def := promptSelection.definition + promptDefinitionHash := promptSelection.hash selectedProfileID := strings.TrimSpace(req.ProfileID) if selectedProfileID == "" {