package promptkit_test import ( "context" "errors" "fmt" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine { t.Helper() engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("requirements contract", map[string]any{ "prompts": prompts, "profiles": profiles, }), }}}, options...) if err != nil { t.Fatalf("construct requirements engine: %v", err) } return engine } type requirementCountingArtifactReader struct{ calls atomic.Int64 } func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { reader.calls.Add(1) return nil, errors.New("artifact reader must not be called") } func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any { definition := map[string]any{ "messages": []any{map[string]any{"role": "user", "content": content}}, "output": output, } if defaultProfile != "" { definition["default_profile"] = defaultProfile } if inputs != nil { definition["inputs"] = inputs } if session != "" { definition["session_id"] = session } return definition } func oneVersionPrompt(id, version string, definition map[string]any) map[string]any { return map[string]any{id: promptDefinitionWithVersion(version, definition)} } func promptDefinitionWithVersion(version string, definition map[string]any) map[string]any { result := make(map[string]any, len(definition)+1) result["version"] = version for key, value := range definition { result[key] = value } return result } var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"} func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) { var modelCalls atomic.Int64 reader := &requirementCountingArtifactReader{} definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{ map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."}, map[string]any{"name": "optional", "content_type": "text/plain", "description": ""}, }, `{{input "session-only"}} {{.session_variable}}`, map[string]any{ "format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json", }) engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{ "partial": map[string]any{"model": "partial-model", "api_key_required": true}, }, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"}, SessionID: "SESSION_PRIVATE_MARKER", }) if err != nil { t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err) } if inspection.PromptID != "requirements" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete { t.Fatalf("selected context or completeness is wrong: %+v", inspection) } inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs)) for _, input := range inspection.Inputs { inputs[input.Name] = input } if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared || inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" || inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." { t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs) } if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" || inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" || inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared || inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared || inputs["optional"].Description == nil || *inputs["optional"].Description != "" { t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs) } if _, found := inputs["session-only"]; found { t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs) } variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables)) for _, variable := range inspection.Variables { variables[variable.Name] = variable } if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied || variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied || variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" { t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables) } if _, found := variables["unrelated"]; found { t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables) } formatted := fmt.Sprintf("%+v", inspection) for _, private := range []string{ "TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER", } { if strings.Contains(formatted, private) { t.Fatalf("inspection disclosed %q: %s", private, formatted) } } if reader.calls.Load() != 0 || modelCalls.Load() != 0 { t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load()) } } func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) { prompts := map[string]any{ "first-prompt": promptDefinitionWithVersion("v1", contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput)), "second-prompt": promptDefinitionWithVersion("v2", contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput)), "without-default": contractPrompt("", "plain", nil, "", noValidationOutput), "direct-required": promptDefinitionWithVersion("v1", contractPrompt("direct", "plain", nil, "", noValidationOutput)), } profiles := map[string]any{ "first": map[string]any{"model": "first-model"}, "second": map[string]any{"model": "second-model"}, "direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true}, } engine := requirementsEngine(t, prompts, profiles) first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt"}) if err != nil || first.PromptVersion == nil || *first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" { t.Fatalf("first prompt ID/default profile=(%+v, %v)", first, err) } second, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ PromptID: "second-prompt", ProfileID: "first", }) if err != nil || second.PromptVersion == nil || *second.PromptVersion != "v2" || second.ProfileID != "first" || second.Inputs[0].Name != "two" { t.Fatalf("second prompt ID/request profile=(%+v, %v)", second, err) } if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "First-Prompt"}); !errors.Is(err, promptkit.ErrPromptNotFound) { t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err) } if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) { t.Fatalf("missing profile error=%v", err) } if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) { t.Fatalf("missing profile selection error=%v", err) } if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" { t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err) } if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) { t.Fatalf("blank prompt error=%v", err) } ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "first-prompt"}); !errors.Is(err, context.Canceled) { t.Fatalf("canceled inspection error=%v", err) } } func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) { prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt( "profile", "Hello {{.name}}", nil, "", noValidationOutput, )) engine := requirementsEngine(t, prompts, map[string]any{ "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"}, }) inspection, err := engine.InspectPrompt(context.Background(), "hash-equality") if err != nil { t.Fatalf("inspect captured prompt: %v", err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "hash-equality", Vars: map[string]string{"name": "Ada"}, }) if err != nil { t.Fatalf("prepare captured prompt: %v", err) } if inspection.PromptHash != prepared.PromptHash { t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash) } } func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) { prompts := map[string]any{ "conditional": promptDefinitionWithVersion("v1", contractPrompt("profile", `{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)), "optional": promptDefinitionWithVersion("v1", contractPrompt("profile", `{{index . "lookup"}}`, []any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)), "empty": promptDefinitionWithVersion("v1", contractPrompt("profile", "plain", nil, "", noValidationOutput)), } engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}}) conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"}) if err != nil || conditional.Complete || len(conditional.Issues) != 1 || conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 || conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional { t.Fatalf("conditional discovery=(%+v, %v)", conditional, err) } optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"}) if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 || optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 || optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional { t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err) } empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"}) if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 { t.Fatalf("empty complete result=(%+v, %v)", empty, err) } } func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) { definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{ map[string]any{"name": "transcript", "required": true}, }, `{{.session_var}} {{input "session-only"}}`, noValidationOutput) engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{ "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"}, }) inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session", }) if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 || inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional || inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired || len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional || inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired { t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session", Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")}, }) if err != nil { t.Fatalf("prepare with matching synthetic artifact: %v", err) } if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" { t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared) } if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ PromptID: "ready", Vars: map[string]string{"name": "Ada"}, }); err != nil { t.Fatalf("inspect without session bypass: %v", err) } } func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) { files := fstest.MapFS{ "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)}, } source := &unreadableAfterConstructionFS{FS: files} engine := capturedPromptEngine(t, source) source.closed.Store(true) delete(files, "config/body.tmpl") inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"}) if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 { t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load()) } } func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) { definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{ map[string]any{"name": "declared", "required": true, "description": "captured description"}, }, `{{.session}}`, noValidationOutput) engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{ "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"}, }) request := promptkit.RequirementInspectionRequest{PromptID: "owned"} first, err := engine.InspectRequirements(context.Background(), request) if err != nil { t.Fatal(err) } if first.PromptVersion == nil || *first.PromptVersion != "v1" { t.Fatalf("requirement inspection label = %v, want v1", first.PromptVersion) } *first.PromptVersion = "mutated label" for index := range first.Inputs { if first.Inputs[index].Name == "declared" { first.Inputs[index].Name = "mutated" if first.Inputs[index].Description != nil { *first.Inputs[index].Description = "mutated description" } } } first.Variables[0].Name = "mutated-variable" first.Issues[0].Description = "mutated issue" second, err := engine.InspectRequirements(context.Background(), request) if err != nil { t.Fatalf("inspect after mutating returned values: %v", err) } var description *string for _, input := range second.Inputs { if input.Name == "declared" { description = input.Description } } if second.PromptVersion == nil || *second.PromptVersion != "v1" || second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" || description == nil || *description != "captured description" || len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" { t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err) } const count = 12 var wait sync.WaitGroup errs := make(chan error, count*2) for index := 0; index < count; index++ { wait.Add(2) go func() { defer wait.Done() _, err := engine.InspectRequirements(context.Background(), request) errs <- err }() go func(index int) { defer wait.Done() _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"}, Inputs: map[string]promptkit.ArtifactRef{ "conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"), }, }) errs <- err }(index) } wait.Wait() close(errs) for err := range errs { if err != nil { t.Fatal(err) } } } func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) { t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker") definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput) engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{ "profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"}, }) inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{ PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"}, SessionID: "direct-session-marker", }) if err != nil { t.Fatal(err) } formatted := fmt.Sprintf("%+v", inspection) for _, marker := range []string{ "template-body-marker", "known-value-marker", "unrelated-value-marker", "direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET", } { if strings.Contains(formatted, marker) { t.Fatalf("inspection disclosed %q: %s", marker, formatted) } } }