From bf472dad8ae9e6918f92c86cf0c2a9da203ce581 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 8 Oct 2026 20:17:34 +0000 Subject: [PATCH] Capture prompt requirement metadata and templates --- docs/internal/sources.md | 17 ++-- docs/roadmap/implementation.md | 4 + internal/configuration/load.go | 53 ++++++++++-- internal/configuration/prompts.go | 88 +++++++++++++++++--- internal/configuration/prompts_test.go | 107 +++++++++++++++++++++++++ internal/prompt/template.go | 16 ++++ 6 files changed, 261 insertions(+), 24 deletions(-) diff --git a/docs/internal/sources.md b/docs/internal/sources.md index e36b1cf..deeba0e 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -35,7 +35,12 @@ The configuration adapter indexes each prompt by exact ID and version, materializes file-backed message content using the exact effective field's origin and resource context, and checks static message/session template syntax at construction. It stores captured content, compiled templates, declared -input metadata, and a definition hash. It closes content-file handles before +input metadata, and a definition hash. Alongside the unchanged hashed +definition, it retains content-type and description presence for each declared +input and provides an internal requirement snapshot with those declarations +and the captured parses. Metadata values come from the final normalized +declaration; absent and null fields remain distinguishable from explicit +strings, including empty strings. It closes content-file handles before returning and retains no source capability for public inspection. Role and cache-control normalization is applied to execution-owned message @@ -43,11 +48,13 @@ copies. The retained ConfigKit configuration keeps effective input message metadata and its resource origins: recording a sequence-wide normalization in ConfigKit v0.5.0 would replace origins of unchanged content-file fields. -The runner selects from this index for preparation and `InspectPrompt`. -Neither path rescans configuration or reopens message files. `InspectPrompt` +The runner selects from this index for preparation and `InspectPrompt`; the +internal requirement snapshot lookup selects by exact prompt ID and version. +These paths do not rescan configuration or reopen message files. `InspectPrompt` returns declared metadata without rendering or resolving profile, input, or -schema resources. Runtime rendering still checks request variables and input -helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go) +schema resources, and its public shape and prompt hashes do not include the +captured presence state. Runtime rendering still checks request variables and +input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go) cover resource origins, context containment, capture, and selection. ## Profiles And Built-Ins diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d54c53b..501b700 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -203,6 +203,10 @@ result is not a rendering/readiness/provider guarantee. ## Stage 1 — Capture Effective Metadata And Template Snapshots +**Status: Complete (2026-10-08).** Effective input metadata presence and +construction-time templates are available through an internal snapshot without +changing prompt hashes or the existing inspection shape. + **Outcome:** The analyzer can obtain existing compiled templates and declaration metadata without artifact reads or changing old inspection/execution behavior. diff --git a/internal/configuration/load.go b/internal/configuration/load.go index c2ad0f5..8b08da2 100644 --- a/internal/configuration/load.go +++ b/internal/configuration/load.go @@ -54,10 +54,10 @@ type promptConfig struct { } type promptInputConfig struct { - Name string `yaml:"name"` - Required bool `yaml:"required"` - ContentType string `yaml:"content_type"` - Description string `yaml:"description"` + Name string `yaml:"name"` + Required bool `yaml:"required"` + ContentType *string `yaml:"content_type"` + Description *string `yaml:"description"` } type promptMessageConfig struct { @@ -93,6 +93,7 @@ type loaded struct { result *configkit.Result[document] backends map[string]map[string]fieldState profiles map[string]map[string]fieldState + promptMetadata map[string]map[string][]promptInputPresence prompts map[string]map[string]*promptEntry ambiguous map[string]bool registry *backend.Registry @@ -112,8 +113,9 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) // load performs one ConfigKit lifecycle over the selected source tiers. func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { state := &loaded{ - backends: make(map[string]map[string]fieldState), - profiles: make(map[string]map[string]fieldState), + backends: make(map[string]map[string]fieldState), + profiles: make(map[string]map[string]fieldState), + promptMetadata: make(map[string]map[string][]promptInputPresence), } result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{ Underlays: underlays, @@ -122,6 +124,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) Resolve: func(config *document, resolution *configkit.Resolution) error { captureFields(config.Backends, "backends", backendFields, resolution, state.backends) captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles) + capturePromptMetadata(config.Prompts, resolution, state.promptMetadata) resolveBackends(config.Backends, resolution) resolveProfiles(config.Profiles, resolution) resolvePrompts(config.Prompts, resolution) @@ -143,6 +146,44 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) return state, nil } +type promptInputPresence struct { + contentType fieldState + description fieldState +} + +func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) { + for _, id := range sortedKeys(prompts) { + if destination[id] == nil { + destination[id] = make(map[string][]promptInputPresence, len(prompts[id])) + } + for _, version := range sortedKeys(prompts[id]) { + inputs := prompts[id][version].Inputs + states := make([]promptInputPresence, len(inputs)) + base := configkit.Keys("prompts", id, version, "inputs") + for index := range inputs { + path := base.Index(index) + states[index] = promptInputPresence{ + contentType: captureFieldState(path.Key("content_type"), resolution), + description: captureFieldState(path.Key("description"), resolution), + } + } + destination[id][version] = states + } + } +} + +func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState { + info, exists := resolution.InputInfo(path) + switch { + case !exists || !info.Explicit: + return fieldAbsent + case info.Null: + return fieldNull + default: + return fieldValue + } +} + func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) { for id := range entries { states := make(map[string]fieldState, len(fields)) diff --git a/internal/configuration/prompts.go b/internal/configuration/prompts.go index 7606830..f783d46 100644 --- a/internal/configuration/prompts.go +++ b/internal/configuration/prompts.go @@ -21,11 +21,12 @@ var ( ) type promptEntry struct { - definition domain.PromptDefinition - inspection domain.PromptInspection - hash string - session *template.Template - messages []*template.Template + definition domain.PromptDefinition + inspection domain.PromptInspection + requirementInputs []prompt.InputMetadata + hash string + session *template.Template + messages []*template.Template } func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) { @@ -48,18 +49,24 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf for index := range definition.Inputs { input := &definition.Inputs[index] inputPath := base.Key("inputs").Index(index) + normalizedName := strings.TrimSpace(input.Name) + if normalizedName != input.Name { + configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata") + inputsChanged = true + } for _, field := range []struct { name string - value *string + value **string }{ - {"name", &input.Name}, {"content_type", &input.ContentType}, {"description", &input.Description}, } { - normalized := strings.TrimSpace(*field.value) - if normalized != *field.value { - configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata") - inputsChanged = true + if *field.value != nil { + normalized := strings.TrimSpace(**field.value) + if normalized != **field.value { + configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata") + inputsChanged = true + } } } } @@ -132,6 +139,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo } seenInputs := make(map[string]bool, len(raw.Inputs)) definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs)) + entry := &promptEntry{ + messages: make([]*template.Template, 0, len(raw.Messages)), + requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)), + } for index, input := range raw.Inputs { path := base.Key("inputs").Index(index) if input.Name == "" { @@ -141,11 +152,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name")) } seenInputs[input.Name] = true + contentType := "" + if input.ContentType != nil { + contentType = *input.ContentType + } + description := "" + if input.Description != nil { + description = *input.Description + } definition.Inputs = append(definition.Inputs, domain.PromptInput{ - Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description, + Name: input.Name, Required: input.Required, ContentType: contentType, Description: description, }) + presence := state.promptMetadata[id][version][index] + metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required} + if presence.contentType == fieldValue { + metadata.ContentType = stringPointer(contentType) + } + if presence.description == fieldValue { + metadata.Description = stringPointer(description) + } + entry.requirementInputs = append(entry.requirementInputs, metadata) } - entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))} definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) for index, message := range raw.Messages { path := base.Key("messages").Index(index) @@ -233,6 +260,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo return entry, nil } +func stringPointer(value string) *string { + copy := value + return © +} + func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) { if strings.TrimSpace(name) == "" { return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath) @@ -329,3 +361,33 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe } return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true } + +// GetRequirementSnapshot returns captured templates and input metadata for +// one exact prompt identity. Returned metadata and slices are independently +// owned; the compiled templates are shared immutable construction-time parses. +func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) { + if state == nil { + return prompt.RequirementSnapshot{}, false + } + entry := state.prompts[id][version] + if entry == nil { + return prompt.RequirementSnapshot{}, false + } + snapshot := prompt.RequirementSnapshot{ + Templates: prompt.CompiledTemplates{ + Session: entry.session, + Messages: append([]*template.Template(nil), entry.messages...), + }, + Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)), + } + for index, input := range entry.requirementInputs { + snapshot.Inputs[index] = input + if input.ContentType != nil { + snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType) + } + if input.Description != nil { + snapshot.Inputs[index].Description = stringPointer(*input.Description) + } + } + return snapshot, true +} diff --git a/internal/configuration/prompts_test.go b/internal/configuration/prompts_test.go index 44f332b..2afced8 100644 --- a/internal/configuration/prompts_test.go +++ b/internal/configuration/prompts_test.go @@ -170,6 +170,113 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T } } +func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) { + state, err := load(mapping("requirement-metadata", map[string]any{ + "prompts": map[string]any{"example": map[string]any{"v1": map[string]any{ + "session_id": "{{.session}}", + "inputs": []any{ + map[string]any{"name": "absent"}, + map[string]any{"name": "null", "content_type": nil, "description": nil}, + map[string]any{"name": "empty", "content_type": "", "description": ""}, + map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "}, + }, + "messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}}, + "output": map[string]any{"format": "text", "validation_mode": "basic"}, + }}}, + }), nil) + if err != nil { + var loadError *configkit.LoadError + if errors.As(err, &loadError) { + t.Fatalf("%v: %+v", err, loadError.Diagnostics()) + } + t.Fatal(err) + } + + snapshot, ok := state.GetRequirementSnapshot("example", "v1") + if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil { + t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok) + } + for _, index := range []int{0, 1} { + if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil { + t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index]) + } + } + if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" || + snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" { + t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2]) + } + if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" || + snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" { + t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3]) + } + + definition, inspection, err := state.prompt("example", "v1") + if err != nil { + t.Fatal(err) + } + if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" || + inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" { + t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection) + } + + *snapshot.Inputs[2].ContentType = "caller mutation" + snapshot.Inputs[3].Name = "caller mutation" + snapshot.Templates.Messages[0] = nil + again, ok := state.GetRequirementSnapshot("example", "v1") + if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil { + t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok) + } + if _, ok := state.GetRequirementSnapshot("example", "V1"); ok { + t.Fatal("snapshot lookup unexpectedly changed exact version matching") + } +} + +func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) { + variants := []struct { + name string + fields map[string]any + }{ + {name: "absent"}, + {name: "null", fields: map[string]any{"content_type": nil, "description": nil}}, + {name: "empty", fields: map[string]any{"content_type": "", "description": ""}}, + } + var expectedHash string + for _, variant := range variants { + t.Run(variant.name, func(t *testing.T) { + input := map[string]any{"name": "document"} + for name, value := range variant.fields { + input[name] = value + } + state, err := load(mapping("hash-"+variant.name, map[string]any{ + "prompts": map[string]any{"same": map[string]any{"v1": map[string]any{ + "inputs": []any{input}, + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "basic"}, + }}}, + }), nil) + if err != nil { + var loadError *configkit.LoadError + if errors.As(err, &loadError) { + t.Fatalf("%v: %+v", err, loadError.Diagnostics()) + } + t.Fatal(err) + } + _, inspection, err := state.prompt("same", "v1") + if err != nil { + t.Fatal(err) + } + if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" { + t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs) + } + if expectedHash == "" { + expectedHash = inspection.PromptHash + } else if inspection.PromptHash != expectedHash { + t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash) + } + }) + } +} + func promptWithFile(name string) map[string]any { return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{ "messages": []any{map[string]any{"role": "user", "content_file": name}}, diff --git a/internal/prompt/template.go b/internal/prompt/template.go index 63bc488..82422b2 100644 --- a/internal/prompt/template.go +++ b/internal/prompt/template.go @@ -9,6 +9,22 @@ type CompiledTemplates struct { Messages []*template.Template } +// InputMetadata is the captured declaration metadata for one prompt input. +// Nil metadata pointers mean the corresponding field was absent or null. +type InputMetadata struct { + Name string + Required bool + ContentType *string + Description *string +} + +// RequirementSnapshot provides the captured syntax and declarations needed +// for requirement inspection without reopening prompt resources. +type RequirementSnapshot struct { + Templates CompiledTemplates + Inputs []InputMetadata +} + // TemplateLookup provides parsed templates for a captured prompt identity. type TemplateLookup interface { GetCompiledTemplates(id, version string) (CompiledTemplates, bool)