From c5f733af4515524f58a18025bca5d04ba625e07e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 00:45:25 +0000 Subject: [PATCH] Select prompt metadata for inspection --- docs/internal/runner.md | 37 +++++------ docs/internal/sources.md | 3 +- internal/usecase/prompt_inspection.go | 57 ++++++++++++----- internal/usecase/prompt_inspection_test.go | 59 +++++++++++++++++ internal/usecase/requirements_inspection.go | 9 +-- .../usecase/requirements_inspection_test.go | 63 +++++++++++++++++++ requirements_inspection_contract_test.go | 22 +++++++ 7 files changed, 212 insertions(+), 38 deletions(-) diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 2b840a8..324871b 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -32,26 +32,27 @@ The runner has no durable run or session store. ## Shared Prompt Selection -The runner uses one prompt-selection and hashing boundary for ordinary -preparation and exact prompt inspection. Preparation retains its early -request-ID check before direct-session normalization; both operations select -one captured definition and its construction-time hash. Referenced message -content and static template syntax were captured when the engine was built. +Ordinary preparation selects the full prompt definition and construction-time +hash. It retains its early request-ID check before direct-session +normalization. Referenced message content and static template syntax were +captured when the engine was built. -Inspection stops after that structural lookup. It does not render templates or -touch profile, artifact, schema, renderer, validator, admission, or model -collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns -the public operation's exact contract. +`InspectPrompt` selects captured metadata through an optional repository +capability, falling back to the full definition and existing hash resolution +for internal repositories without it. It returns owned metadata without +rendering templates or touching profile, artifact, schema, renderer, validator, +admission, or model collaborators. The root +[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's +exact contract. -`InspectRequirements` uses the same prompt selection boundary, then selects the -request profile or prompt default through an optional identity-only repository -capability, then obtains the exact captured requirement snapshot. Internal -repositories without that capability use the full-profile selection boundary; -both paths preserve the exact selected ID, and the fallback retains its -profile/target validity checks. It analyzes the snapshot with the supplied -caller variables and normalized direct-session bypass; it does not resolve -target readiness or enter the preparation pipeline. Missing snapshot -capability is a prompt-load failure rather than a source fallback. The root +`InspectRequirements` uses the same metadata selection to resolve the +request-profile/default identity, then obtains the exact captured requirement +snapshot by the selected prompt ID and version. The analyzer receives only +those identity fields rather than an execution definition. It analyzes the +snapshot with the supplied caller variables and normalized direct-session +bypass; it does not resolve target readiness or enter the preparation +pipeline. Missing snapshot capability is a prompt-load failure rather than a +source fallback. The root [`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public operation's exact contract. diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 5c71896..a551341 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -48,7 +48,8 @@ copies. The retained ConfigKit configuration keeps effective input message metadata and its resource origins: recording a sequence-wide normalization in ConfigKit v0.5.0 would replace origins of unchanged content-file fields. -The runner selects from this index for preparation and `InspectPrompt`; the +The runner selects full definitions from this index for preparation and uses +captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the internal requirement snapshot lookup selects by exact prompt ID and version. These paths do not rescan configuration or reopen message files. `InspectPrompt` returns declared metadata without rendering or resolving profile, input, or diff --git a/internal/usecase/prompt_inspection.go b/internal/usecase/prompt_inspection.go index 4ee74a9..f985a8b 100644 --- a/internal/usecase/prompt_inspection.go +++ b/internal/usecase/prompt_inspection.go @@ -13,6 +13,10 @@ type resolvedPromptDefinition struct { hash string } +type promptInspectionRepository interface { + GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error) +} + func (r *Runner) resolvePromptDefinition( ctx context.Context, promptID string, @@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition( return &resolvedPromptDefinition{definition: definition, hash: hash}, nil } +func (r *Runner) resolvePromptInspection( + ctx context.Context, + promptID string, + promptVersion string, +) (*domain.PromptInspection, error) { + if strings.TrimSpace(promptID) == "" { + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) + } + if r == nil || r.promptDefs == nil { + return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad) + } + if indexed, ok := r.promptDefs.(promptInspectionRepository); ok { + inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) + } + if inspection == nil { + return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad) + } + return inspection, nil + } + + selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion) + if err != nil { + return nil, err + } + definition := selection.definition + return &domain.PromptInspection{ + PromptID: definition.ID, + PromptVersion: definition.Version, + PromptHash: selection.hash, + DefaultProfileID: definition.DefaultProfile, + Inputs: append([]domain.PromptInput(nil), definition.Inputs...), + OutputContract: definition.Validation, + }, nil +} + // InspectPrompt resolves one explicit prompt without execution work. func (r *Runner) InspectPrompt( ctx context.Context, @@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt( default: } - selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion) - if err != nil { - return nil, err - } - inputs := make([]domain.PromptInput, len(selection.definition.Inputs)) - copy(inputs, selection.definition.Inputs) - - return &domain.PromptInspection{ - PromptID: selection.definition.ID, - PromptVersion: selection.definition.Version, - PromptHash: selection.hash, - DefaultProfileID: selection.definition.DefaultProfile, - Inputs: inputs, - OutputContract: selection.definition.Validation, - }, nil + return r.resolvePromptInspection(ctx, promptID, promptVersion) } diff --git a/internal/usecase/prompt_inspection_test.go b/internal/usecase/prompt_inspection_test.go index c26acdc..e87c3c8 100644 --- a/internal/usecase/prompt_inspection_test.go +++ b/internal/usecase/prompt_inspection_test.go @@ -18,6 +18,13 @@ type inspectionPromptRepository struct { version string } +type promptMetadataRepository struct { + inspection domain.PromptInspection + inspectionErr error + lookupID string + lookupVersion string +} + func (r *inspectionPromptRepository) GetPromptDefinition( _ context.Context, id string, @@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition( return r.definition, nil } +func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) { + return nil, errors.New("full prompt definition should not be requested") +} + +func (r *promptMetadataRepository) GetPromptInspection( + _ context.Context, + id string, + version string, +) (*domain.PromptInspection, error) { + r.lookupID = id + r.lookupVersion = version + inspection := r.inspection + inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...) + return &inspection, r.inspectionErr +} + func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) { definition := &domain.PromptDefinition{ ID: "normalized.prompt", @@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t } } +func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) { + repository := &promptMetadataRepository{inspection: domain.PromptInspection{ + PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash", + DefaultProfileID: "partial-profile", + Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}}, + OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON}, + }} + runner := &Runner{promptDefs: repository} + inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ") + if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " || + inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" || + inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" || + !reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract { + t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion) + } + inspection.Inputs[0].Name = "caller mutation" + inspection.OutputContract.SchemaPath = "caller mutation" + second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version") + if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" { + t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err) + } +} + +func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) { + repository := &requirementInspectionPromptRepository{ + definition: &domain.PromptDefinition{ + ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}}, + }, + } + inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ") + if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" || + inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" { + t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err) + } +} + func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) { t.Run("blank ID", func(t *testing.T) { repository := &inspectionPromptRepository{} diff --git a/internal/usecase/requirements_inspection.go b/internal/usecase/requirements_inspection.go index fcb36e4..c243ac5 100644 --- a/internal/usecase/requirements_inspection.go +++ b/internal/usecase/requirements_inspection.go @@ -40,7 +40,7 @@ func (r *Runner) InspectRequirements( return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) } - selection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion) + selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func (r *Runner) InspectRequirements( } selectedProfileID := req.ProfileID if strings.TrimSpace(selectedProfileID) == "" { - selectedProfileID = selection.definition.DefaultProfile + selectedProfileID = selection.DefaultProfileID } if strings.TrimSpace(selectedProfileID) == "" { return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired) @@ -66,11 +66,12 @@ func (r *Runner) InspectRequirements( if !ok { return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad) } - snapshot, ok := repository.GetRequirementSnapshot(selection.definition.ID, selection.definition.Version) + snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion) if !ok { return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad) } - inspection, err := prompt.AnalyzeRequirements(ctx, selection.definition, snapshot, req.Vars, directSessionID != "") + definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion} + inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "") if err != nil { return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) } diff --git a/internal/usecase/requirements_inspection_test.go b/internal/usecase/requirements_inspection_test.go index 9f8f6cc..0a8ffa4 100644 --- a/internal/usecase/requirements_inspection_test.go +++ b/internal/usecase/requirements_inspection_test.go @@ -24,6 +24,13 @@ type requirementInspectionPromptRepository struct { snapshotVersion string } +type requirementPromptMetadataRepository struct { + *requirementInspectionPromptRepository + inspection domain.PromptInspection + lookupID string + lookupVersion string +} + type inspectionProfileIdentityRepository struct { *inspectionProfileRepository identity string @@ -49,6 +56,18 @@ func (repository *requirementInspectionPromptRepository) GetPromptDefinition( return repository.definition, repository.lookupErr } +func (repository *requirementPromptMetadataRepository) GetPromptInspection( + _ context.Context, + id string, + version string, +) (*domain.PromptInspection, error) { + repository.lookupID = id + repository.lookupVersion = version + inspection := repository.inspection + inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...) + return &inspection, nil +} + func (repository *requirementInspectionPromptRepository) GetPromptSnapshot( _ context.Context, id string, @@ -111,6 +130,50 @@ func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *t } } +func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) { + message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`) + if err != nil { + t.Fatal(err) + } + session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`) + if err != nil { + t.Fatal(err) + } + baseRepository := &requirementInspectionPromptRepository{ + snapshotFound: true, + snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{ + Messages: []*template.Template{message}, Session: session, + }}, + } + promptRepository := &requirementPromptMetadataRepository{ + requirementInspectionPromptRepository: baseRepository, + inspection: domain.PromptInspection{ + PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash", + DefaultProfileID: "default-profile", + }, + } + profiles := &inspectionProfileIdentityRepository{ + inspectionProfileRepository: &inspectionProfileRepository{}, + identity: "default-profile", + } + inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements( + context.Background(), RequirementInspectionRequest{ + PromptID: " requested-id ", PromptVersion: " requested-version ", + Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ", + }, + ) + if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " || + baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" || + profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" || + inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" || + !inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || + len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied { + t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q", + inspection, err, promptRepository.lookupID, promptRepository.lookupVersion, + baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID) + } +} + func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) { message, err := prompt.CompileTemplate("message", `{{.name}}`) if err != nil { diff --git a/requirements_inspection_contract_test.go b/requirements_inspection_contract_test.go index 9b02f54..bae54eb 100644 --- a/requirements_inspection_contract_test.go +++ b/requirements_inspection_contract_test.go @@ -178,6 +178,28 @@ func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *test } } +func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) { + prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt( + "profile", "Hello {{.name}}", nil, "", noValidationOutput, + )) + engine := requirementsEngine(t, prompts, map[string]any{ + "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"}, + }) + inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1") + if err != nil { + t.Fatalf("inspect captured prompt: %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"}, + }) + if err != nil { + t.Fatalf("prepare captured prompt: %v", err) + } + if inspection.PromptHash != prepared.PromptHash { + t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash) + } +} + func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) { prompts := map[string]any{ "conditional": map[string]any{"v1": contractPrompt("profile",