eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Select prompt metadata for inspection

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-09 00:45:25 +00:00
parent c09d2c6e20
commit c5f733af45
7 changed files with 212 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
docs/internal/runner.md
View File

@@ -32,26 +32,27 @@ The runner has no durable run or session store.
## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations select
one captured definition and its construction-time hash. Referenced message
content and static template syntax were captured when the engine was built.
Ordinary preparation selects the full prompt definition and construction-time
hash. It retains its early request-ID check before direct-session
normalization. Referenced message content and static template syntax were
captured when the engine was built.
Inspection stops after that structural lookup. It does not render templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
`InspectPrompt` selects captured metadata through an optional repository
capability, falling back to the full definition and existing hash resolution
for internal repositories without it. It returns owned metadata without
rendering templates or touching profile, artifact, schema, renderer, validator,
admission, or model collaborators. The root
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
exact contract.
`InspectRequirements` uses the same prompt selection boundary, then selects the
request profile or prompt default through an optional identity-only repository
capability, then obtains the exact captured requirement snapshot. Internal
repositories without that capability use the full-profile selection boundary;
both paths preserve the exact selected ID, and the fallback retains its
profile/target validity checks. It analyzes the snapshot with the supplied
caller variables and normalized direct-session bypass; it does not resolve
target readiness or enter the preparation pipeline. Missing snapshot
capability is a prompt-load failure rather than a source fallback. The root
`InspectRequirements` uses the same metadata selection to resolve the
request-profile/default identity, then obtains the exact captured requirement
snapshot by the selected prompt ID and version. The analyzer receives only
those identity fields rather than an execution definition. It analyzes the
snapshot with the supplied caller variables and normalized direct-session
bypass; it does not resolve target readiness or enter the preparation
pipeline. Missing snapshot capability is a prompt-load failure rather than a
source fallback. The root
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
operation's exact contract.

3
docs/internal/sources.md
View File

@@ -48,7 +48,8 @@ copies. The retained ConfigKit configuration keeps effective input message
metadata and its resource origins: recording a sequence-wide normalization in
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
The runner selects from this index for preparation and `InspectPrompt`; the
The runner selects full definitions from this index for preparation and uses
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
internal requirement snapshot lookup selects by exact prompt ID and version.
These paths do not rescan configuration or reopen message files. `InspectPrompt`
returns declared metadata without rendering or resolving profile, input, or

57
internal/usecase/prompt_inspection.go
View File

@@ -13,6 +13,10 @@ type resolvedPromptDefinition struct {
hash string
}
type promptInspectionRepository interface {
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
}
func (r *Runner) resolvePromptDefinition(
ctx context.Context,
promptID string,
@@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition(
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}
func (r *Runner) resolvePromptInspection(
ctx context.Context,
promptID string,
promptVersion string,
) (*domain.PromptInspection, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if inspection == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
}
return inspection, nil
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
definition := selection.definition
return &domain.PromptInspection{
PromptID: definition.ID,
PromptVersion: definition.Version,
PromptHash: selection.hash,
DefaultProfileID: definition.DefaultProfile,
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: definition.Validation,
}, nil
}
// InspectPrompt resolves one explicit prompt without execution work.
func (r *Runner) InspectPrompt(
ctx context.Context,
@@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt(
default:
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
copy(inputs, selection.definition.Inputs)
return &domain.PromptInspection{
PromptID: selection.definition.ID,
PromptVersion: selection.definition.Version,
PromptHash: selection.hash,
DefaultProfileID: selection.definition.DefaultProfile,
Inputs: inputs,
OutputContract: selection.definition.Validation,
}, nil
return r.resolvePromptInspection(ctx, promptID, promptVersion)
}

59
internal/usecase/prompt_inspection_test.go
View File

@@ -18,6 +18,13 @@ type inspectionPromptRepository struct {
version string
}
type promptMetadataRepository struct {
inspection domain.PromptInspection
inspectionErr error
lookupID string
lookupVersion string
}
func (r *inspectionPromptRepository) GetPromptDefinition(
_ context.Context,
id string,
@@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition(
return r.definition, nil
}
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
return nil, errors.New("full prompt definition should not be requested")
}
func (r *promptMetadataRepository) GetPromptInspection(
_ context.Context,
id string,
version string,
) (*domain.PromptInspection, error) {
r.lookupID = id
r.lookupVersion = version
inspection := r.inspection
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &inspection, r.inspectionErr
}
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
definition := &domain.PromptDefinition{
ID: "normalized.prompt",
@@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
}
}
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
DefaultProfileID: "partial-profile",
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
}}
runner := &Runner{promptDefs: repository}
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
}
inspection.Inputs[0].Name = "caller mutation"
inspection.OutputContract.SchemaPath = "caller mutation"
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
}
}
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
repository := &requirementInspectionPromptRepository{
definition: &domain.PromptDefinition{
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
},
}
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
}
}
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
repository := &inspectionPromptRepository{}

9
internal/usecase/requirements_inspection.go
View File

@@ -40,7 +40,7 @@ func (r *Runner) InspectRequirements(
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
selection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, err
}
@@ -49,7 +49,7 @@ func (r *Runner) InspectRequirements(
}
selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" {
selectedProfileID = selection.definition.DefaultProfile
selectedProfileID = selection.DefaultProfileID
}
if strings.TrimSpace(selectedProfileID) == "" {
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
@@ -66,11 +66,12 @@ func (r *Runner) InspectRequirements(
if !ok {
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
}
snapshot, ok := repository.GetRequirementSnapshot(selection.definition.ID, selection.definition.Version)
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
if !ok {
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
}
inspection, err := prompt.AnalyzeRequirements(ctx, selection.definition, snapshot, req.Vars, directSessionID != "")
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}

63
internal/usecase/requirements_inspection_test.go
View File

@@ -24,6 +24,13 @@ type requirementInspectionPromptRepository struct {
snapshotVersion string
}
type requirementPromptMetadataRepository struct {
*requirementInspectionPromptRepository
inspection domain.PromptInspection
lookupID string
lookupVersion string
}
type inspectionProfileIdentityRepository struct {
*inspectionProfileRepository
identity string
@@ -49,6 +56,18 @@ func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
return repository.definition, repository.lookupErr
}
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
_ context.Context,
id string,
version string,
) (*domain.PromptInspection, error) {
repository.lookupID = id
repository.lookupVersion = version
inspection := repository.inspection
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &inspection, nil
}
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
_ context.Context,
id string,
@@ -111,6 +130,50 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t
}
}
func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
if err != nil {
t.Fatal(err)
}
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
if err != nil {
t.Fatal(err)
}
baseRepository := &requirementInspectionPromptRepository{
snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
Messages: []*template.Template{message}, Session: session,
}},
}
promptRepository := &requirementPromptMetadataRepository{
requirementInspectionPromptRepository: baseRepository,
inspection: domain.PromptInspection{
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
DefaultProfileID: "default-profile",
},
}
profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{},
identity: "default-profile",
}
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{
PromptID: " requested-id ", PromptVersion: " requested-version ",
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
},
)
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
}
}
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{.name}}`)
if err != nil {

22
requirements_inspection_contract_test.go
View File

@@ -178,6 +178,28 @@ func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *test
}
}
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
))
engine := requirementsEngine(t, prompts, map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
})
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1")
if err != nil {
t.Fatalf("inspect captured prompt: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
})
if err != nil {
t.Fatalf("prepare captured prompt: %v", err)
}
if inspection.PromptHash != prepared.PromptHash {
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
}
}
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
prompts := map[string]any{
"conditional": map[string]any{"v1": contractPrompt("profile",