370 lines
18 KiB
Go
370 lines
18 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
|
t.Helper()
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
|
Source: configkit.Mapping("requirements contract", map[string]any{
|
|
"prompts": prompts, "profiles": profiles,
|
|
}),
|
|
}}}, options...)
|
|
if err != nil {
|
|
t.Fatalf("construct requirements engine: %v", err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
type requirementCountingArtifactReader struct{ calls atomic.Int64 }
|
|
|
|
func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
|
reader.calls.Add(1)
|
|
return nil, errors.New("artifact reader must not be called")
|
|
}
|
|
|
|
func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any {
|
|
definition := map[string]any{
|
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
|
"output": output,
|
|
}
|
|
if defaultProfile != "" {
|
|
definition["default_profile"] = defaultProfile
|
|
}
|
|
if inputs != nil {
|
|
definition["inputs"] = inputs
|
|
}
|
|
if session != "" {
|
|
definition["session_id"] = session
|
|
}
|
|
return definition
|
|
}
|
|
|
|
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
|
|
return map[string]any{id: promptDefinitionWithVersion(version, definition)}
|
|
}
|
|
|
|
func promptDefinitionWithVersion(version string, definition map[string]any) map[string]any {
|
|
result := make(map[string]any, len(definition)+1)
|
|
result["version"] = version
|
|
for key, value := range definition {
|
|
result[key] = value
|
|
}
|
|
return result
|
|
}
|
|
|
|
var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"}
|
|
|
|
func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
reader := &requirementCountingArtifactReader{}
|
|
definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{
|
|
map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."},
|
|
map[string]any{"name": "optional", "content_type": "text/plain", "description": ""},
|
|
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
|
|
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
|
|
})
|
|
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
|
|
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
|
|
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
|
PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"},
|
|
SessionID: "SESSION_PRIVATE_MARKER",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
|
|
}
|
|
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
|
|
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
|
|
}
|
|
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
|
|
for _, input := range inspection.Inputs {
|
|
inputs[input.Name] = input
|
|
}
|
|
if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared ||
|
|
inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" ||
|
|
inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." {
|
|
t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs)
|
|
}
|
|
if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" ||
|
|
inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" ||
|
|
inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared ||
|
|
inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared ||
|
|
inputs["optional"].Description == nil || *inputs["optional"].Description != "" {
|
|
t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs)
|
|
}
|
|
if _, found := inputs["session-only"]; found {
|
|
t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs)
|
|
}
|
|
variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables))
|
|
for _, variable := range inspection.Variables {
|
|
variables[variable.Name] = variable
|
|
}
|
|
if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied ||
|
|
variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied ||
|
|
variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" {
|
|
t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables)
|
|
}
|
|
if _, found := variables["unrelated"]; found {
|
|
t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables)
|
|
}
|
|
formatted := fmt.Sprintf("%+v", inspection)
|
|
for _, private := range []string{
|
|
"TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER",
|
|
} {
|
|
if strings.Contains(formatted, private) {
|
|
t.Fatalf("inspection disclosed %q: %s", private, formatted)
|
|
}
|
|
}
|
|
if reader.calls.Load() != 0 || modelCalls.Load() != 0 {
|
|
t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) {
|
|
prompts := map[string]any{
|
|
"first-prompt": promptDefinitionWithVersion("v1", contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput)),
|
|
"second-prompt": promptDefinitionWithVersion("v2", contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput)),
|
|
"without-default": contractPrompt("", "plain", nil, "", noValidationOutput),
|
|
"direct-required": promptDefinitionWithVersion("v1", contractPrompt("direct", "plain", nil, "", noValidationOutput)),
|
|
}
|
|
profiles := map[string]any{
|
|
"first": map[string]any{"model": "first-model"},
|
|
"second": map[string]any{"model": "second-model"},
|
|
"direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true},
|
|
}
|
|
engine := requirementsEngine(t, prompts, profiles)
|
|
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt"})
|
|
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
|
|
t.Fatalf("first prompt ID/default profile=(%+v, %v)", first, err)
|
|
}
|
|
second, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
|
PromptID: "second-prompt", ProfileID: "first",
|
|
})
|
|
if err != nil || second.PromptVersion != "v2" || second.ProfileID != "first" || second.Inputs[0].Name != "two" {
|
|
t.Fatalf("second prompt ID/request profile=(%+v, %v)", second, err)
|
|
}
|
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "First-Prompt"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
|
|
t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err)
|
|
}
|
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
|
|
t.Fatalf("missing profile error=%v", err)
|
|
}
|
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
t.Fatalf("missing profile selection error=%v", err)
|
|
}
|
|
if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" {
|
|
t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err)
|
|
}
|
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
t.Fatalf("blank prompt error=%v", err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "first-prompt"}); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("canceled inspection error=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
|
|
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
|
|
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
|
|
))
|
|
engine := requirementsEngine(t, prompts, map[string]any{
|
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
|
|
})
|
|
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality")
|
|
if err != nil {
|
|
t.Fatalf("inspect captured prompt: %v", err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: "hash-equality", Vars: map[string]string{"name": "Ada"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare captured prompt: %v", err)
|
|
}
|
|
if inspection.PromptHash != prepared.PromptHash {
|
|
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
|
|
prompts := map[string]any{
|
|
"conditional": promptDefinitionWithVersion("v1", contractPrompt("profile",
|
|
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)),
|
|
"optional": promptDefinitionWithVersion("v1", contractPrompt("profile", `{{index . "lookup"}}`,
|
|
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)),
|
|
"empty": promptDefinitionWithVersion("v1", contractPrompt("profile", "plain", nil, "", noValidationOutput)),
|
|
}
|
|
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
|
|
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
|
|
if err != nil || conditional.Complete || len(conditional.Issues) != 1 ||
|
|
conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 ||
|
|
conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional {
|
|
t.Fatalf("conditional discovery=(%+v, %v)", conditional, err)
|
|
}
|
|
optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"})
|
|
if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 ||
|
|
optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 ||
|
|
optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional {
|
|
t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err)
|
|
}
|
|
empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"})
|
|
if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 {
|
|
t.Fatalf("empty complete result=(%+v, %v)", empty, err)
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) {
|
|
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
|
|
map[string]any{"name": "transcript", "required": true},
|
|
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
|
|
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
|
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
|
})
|
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
|
})
|
|
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 ||
|
|
inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional ||
|
|
inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired ||
|
|
len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional ||
|
|
inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired {
|
|
t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
|
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare with matching synthetic artifact: %v", err)
|
|
}
|
|
if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" {
|
|
t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared)
|
|
}
|
|
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
|
PromptID: "ready", Vars: map[string]string{"name": "Ada"},
|
|
}); err != nil {
|
|
t.Fatalf("inspect without session bypass: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) {
|
|
files := fstest.MapFS{
|
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
|
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)},
|
|
}
|
|
source := &unreadableAfterConstructionFS{FS: files}
|
|
engine := capturedPromptEngine(t, source)
|
|
source.closed.Store(true)
|
|
delete(files, "config/body.tmpl")
|
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"})
|
|
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 ||
|
|
inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 {
|
|
t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load())
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) {
|
|
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
|
|
map[string]any{"name": "declared", "required": true, "description": "captured description"},
|
|
}, `{{.session}}`, noValidationOutput)
|
|
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
|
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
|
|
})
|
|
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
|
|
first, err := engine.InspectRequirements(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index := range first.Inputs {
|
|
if first.Inputs[index].Name == "declared" {
|
|
first.Inputs[index].Name = "mutated"
|
|
if first.Inputs[index].Description != nil {
|
|
*first.Inputs[index].Description = "mutated description"
|
|
}
|
|
}
|
|
}
|
|
first.Variables[0].Name = "mutated-variable"
|
|
first.Issues[0].Description = "mutated issue"
|
|
second, err := engine.InspectRequirements(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("inspect after mutating returned values: %v", err)
|
|
}
|
|
var description *string
|
|
for _, input := range second.Inputs {
|
|
if input.Name == "declared" {
|
|
description = input.Description
|
|
}
|
|
}
|
|
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
|
|
description == nil || *description != "captured description" ||
|
|
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
|
|
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
|
|
}
|
|
|
|
const count = 12
|
|
var wait sync.WaitGroup
|
|
errs := make(chan error, count*2)
|
|
for index := 0; index < count; index++ {
|
|
wait.Add(2)
|
|
go func() {
|
|
defer wait.Done()
|
|
_, err := engine.InspectRequirements(context.Background(), request)
|
|
errs <- err
|
|
}()
|
|
go func(index int) {
|
|
defer wait.Done()
|
|
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"},
|
|
Inputs: map[string]promptkit.ArtifactRef{
|
|
"conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"),
|
|
},
|
|
})
|
|
errs <- err
|
|
}(index)
|
|
}
|
|
wait.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
|
|
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
|
|
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
|
|
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
|
|
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
|
|
})
|
|
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
|
PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"},
|
|
SessionID: "direct-session-marker",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
formatted := fmt.Sprintf("%+v", inspection)
|
|
for _, marker := range []string{
|
|
"template-body-marker", "known-value-marker", "unrelated-value-marker",
|
|
"direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET",
|
|
} {
|
|
if strings.Contains(formatted, marker) {
|
|
t.Fatalf("inspection disclosed %q: %s", marker, formatted)
|
|
}
|
|
}
|
|
}
|