From ee0ad99e17ceb9b2ec8ff017bca1c487d07d51b7 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 8 Oct 2026 20:43:48 +0000 Subject: [PATCH] Test public prompt requirement inspection --- docs/roadmap/implementation.md | 4 + requirements_inspection_contract_test.go | 348 +++++++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 requirements_inspection_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 46e7c99..d20bb1c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -378,6 +378,10 @@ wiring; commit. ## Stage 6 — Protect Public Inventory And Ownership +**Status: Complete (2026-10-08).** External-package tests cover the assembled +inspection contract, captured-state ownership, and artifact-free use with real +configured engines. + **Outcome:** Public contract coverage protects the consumer need without duplicating the syntax walker's detailed cases. diff --git a/requirements_inspection_contract_test.go b/requirements_inspection_contract_test.go new file mode 100644 index 0000000..9b02f54 --- /dev/null +++ b/requirements_inspection_contract_test.go @@ -0,0 +1,348 @@ +package promptkit_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("requirements contract", map[string]any{ + "prompts": prompts, "profiles": profiles, + }), + }}}, options...) + if err != nil { + t.Fatalf("construct requirements engine: %v", err) + } + return engine +} + +type requirementCountingArtifactReader struct{ calls atomic.Int64 } + +func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + reader.calls.Add(1) + return nil, errors.New("artifact reader must not be called") +} + +func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any { + definition := map[string]any{ + "messages": []any{map[string]any{"role": "user", "content": content}}, + "output": output, + } + if defaultProfile != "" { + definition["default_profile"] = defaultProfile + } + if inputs != nil { + definition["inputs"] = inputs + } + if session != "" { + definition["session_id"] = session + } + return definition +} + +func oneVersionPrompt(id, version string, definition map[string]any) map[string]any { + return map[string]any{id: map[string]any{version: definition}} +} + +var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"} + +func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) { + var modelCalls atomic.Int64 + reader := &requirementCountingArtifactReader{} + definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{ + map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."}, + map[string]any{"name": "optional", "content_type": "text/plain", "description": ""}, + }, `{{input "session-only"}} {{.session_variable}}`, map[string]any{ + "format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json", + }) + engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{ + "partial": map[string]any{"model": "partial-model", "api_key_required": true}, + }, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) + inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ + PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"}, + SessionID: "SESSION_PRIVATE_MARKER", + }) + if err != nil { + t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err) + } + if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete { + t.Fatalf("selected context or completeness is wrong: %+v", inspection) + } + inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs)) + for _, input := range inspection.Inputs { + inputs[input.Name] = input + } + if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared || + inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" || + inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." { + t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs) + } + if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" || + inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" || + inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared || + inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared || + inputs["optional"].Description == nil || *inputs["optional"].Description != "" { + t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs) + } + if _, found := inputs["session-only"]; found { + t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs) + } + variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables)) + for _, variable := range inspection.Variables { + variables[variable.Name] = variable + } + if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied || + variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied || + variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" { + t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables) + } + if _, found := variables["unrelated"]; found { + t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables) + } + formatted := fmt.Sprintf("%+v", inspection) + for _, private := range []string{ + "TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER", + } { + if strings.Contains(formatted, private) { + t.Fatalf("inspection disclosed %q: %s", private, formatted) + } + } + if reader.calls.Load() != 0 || modelCalls.Load() != 0 { + t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load()) + } +} + +func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *testing.T) { + prompts := map[string]any{ + "multi": map[string]any{ + "v1": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput), + "v2": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput), + }, + "without-default": map[string]any{ + "v1": contractPrompt("", "plain", nil, "", noValidationOutput), + }, + "direct-required": map[string]any{"v1": contractPrompt("direct", "plain", nil, "", noValidationOutput)}, + } + profiles := map[string]any{ + "first": map[string]any{"model": "first-model"}, + "second": map[string]any{"model": "second-model"}, + "direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true}, + } + engine := requirementsEngine(t, prompts, profiles) + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi"}); !errors.Is(err, promptkit.ErrPromptLoad) { + t.Fatalf("ambiguous omitted version error=%v", err) + } + first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}) + if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" { + t.Fatalf("exact first version/default profile=(%+v, %v)", first, err) + } + explicit, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ + PromptID: "multi", PromptVersion: "v2", ProfileID: "first", + }) + if err != nil || explicit.PromptVersion != "v2" || explicit.ProfileID != "first" || explicit.Inputs[0].Name != "two" { + t.Fatalf("exact version/request profile=(%+v, %v)", explicit, err) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "Multi", PromptVersion: "v1"}); !errors.Is(err, promptkit.ErrPromptNotFound) { + t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "V1"}); !errors.Is(err, promptkit.ErrPromptNotFound) { + t.Fatalf("case-folded version unexpectedly selected: %v", err) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) { + t.Fatalf("missing profile error=%v", err) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("missing profile selection error=%v", err) + } + if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" { + t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("blank prompt error=%v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled inspection error=%v", err) + } +} + +func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) { + prompts := map[string]any{ + "conditional": map[string]any{"v1": contractPrompt("profile", + `{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)}, + "optional": map[string]any{"v1": contractPrompt("profile", `{{index . "lookup"}}`, + []any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)}, + "empty": map[string]any{"v1": contractPrompt("profile", "plain", nil, "", noValidationOutput)}, + } + engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}}) + conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"}) + if err != nil || conditional.Complete || len(conditional.Issues) != 1 || + conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 || + conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional { + t.Fatalf("conditional discovery=(%+v, %v)", conditional, err) + } + optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"}) + if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 || + optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 || + optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional { + t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err) + } + empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"}) + if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 { + t.Fatalf("empty complete result=(%+v, %v)", empty, err) + } +} + +func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) { + definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{ + map[string]any{"name": "transcript", "required": true}, + }, `{{.session_var}} {{input "session-only"}}`, noValidationOutput) + engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{ + "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"}, + }) + inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ + PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session", + }) + if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 || + inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional || + inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired || + len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional || + inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired { + t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session", + Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")}, + }) + if err != nil { + t.Fatalf("prepare with matching synthetic artifact: %v", err) + } + if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" { + t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared) + } + if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ + PromptID: "ready", Vars: map[string]string{"name": "Ada"}, + }); err != nil { + t.Fatalf("inspect without session bypass: %v", err) + } +} + +func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) { + files := fstest.MapFS{ + "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, + "config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)}, + } + source := &unreadableAfterConstructionFS{FS: files} + engine := capturedPromptEngine(t, source) + source.closed.Store(true) + delete(files, "config/body.tmpl") + inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"}) + if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 || + inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 { + t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load()) + } +} + +func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) { + definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{ + map[string]any{"name": "declared", "required": true, "description": "captured description"}, + }, `{{.session}}`, noValidationOutput) + engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{ + "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"}, + }) + request := promptkit.RequirementInspectionRequest{PromptID: "owned"} + first, err := engine.InspectRequirements(context.Background(), request) + if err != nil { + t.Fatal(err) + } + for index := range first.Inputs { + if first.Inputs[index].Name == "declared" { + first.Inputs[index].Name = "mutated" + if first.Inputs[index].Description != nil { + *first.Inputs[index].Description = "mutated description" + } + } + } + first.Variables[0].Name = "mutated-variable" + first.Issues[0].Description = "mutated issue" + second, err := engine.InspectRequirements(context.Background(), request) + if err != nil { + t.Fatalf("inspect after mutating returned values: %v", err) + } + var description *string + for _, input := range second.Inputs { + if input.Name == "declared" { + description = input.Description + } + } + if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" || + description == nil || *description != "captured description" || + len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" { + t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err) + } + + const count = 12 + var wait sync.WaitGroup + errs := make(chan error, count*2) + for index := 0; index < count; index++ { + wait.Add(2) + go func() { + defer wait.Done() + _, err := engine.InspectRequirements(context.Background(), request) + errs <- err + }() + go func(index int) { + defer wait.Done() + _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"}, + Inputs: map[string]promptkit.ArtifactRef{ + "conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"), + }, + }) + errs <- err + }(index) + } + wait.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } +} + +func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) { + t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker") + definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput) + engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{ + "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"}, + }) + inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ + PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"}, + SessionID: "direct-session-marker", + }) + if err != nil { + t.Fatal(err) + } + formatted := fmt.Sprintf("%+v", inspection) + for _, marker := range []string{ + "template-body-marker", "known-value-marker", "unrelated-value-marker", + "direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET", + } { + if strings.Contains(formatted, marker) { + t.Fatalf("inspection disclosed %q: %s", marker, formatted) + } + } +}