672 lines
34 KiB
Go
672 lines
34 KiB
Go
package prompt
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"text/template"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
)
|
|
|
|
func TestAnalyzeRequirementsDoesNotHideCallerMapScopes(t *testing.T) {
|
|
for _, source := range []string{
|
|
`{{with .}}{{.audience}}{{end}}`,
|
|
`{{with $}}{{.audience}}{{end}}`,
|
|
`{{with (.)}}{{.audience}}{{end}}`,
|
|
`{{$alias := .}}{{with $alias}}{{.audience}}{{end}}`,
|
|
`{{with .}}{{index . "audience"}}{{end}}`,
|
|
`{{$alias := .}}{{index $alias "audience"}}`,
|
|
`{{$alias := .}}{{index ($alias) "audience"}}`,
|
|
`{{$alias := .}}{{$alias.audience}}`,
|
|
} {
|
|
t.Run(source, func(t *testing.T) {
|
|
compiled := compileRequirementTemplate(t, "scope", source)
|
|
vars := map[string]string{"audience": "visible"}
|
|
var rendered strings.Builder
|
|
if err := compiled.Execute(&rendered, vars); err != nil || rendered.String() != "visible" {
|
|
t.Fatalf("caller-map reference did not render: %q, %v", rendered.String(), err)
|
|
}
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
|
}, vars, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Complete {
|
|
if len(result.Variables) != 1 || result.Variables[0].Name != "audience" {
|
|
t.Fatalf("complete discovery lost the caller key: %+v", result)
|
|
}
|
|
} else if len(result.Issues) == 0 {
|
|
t.Fatalf("partial discovery did not explain its limitation: %+v", result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsUnknownScopesAffectOptionalEntriesOnlyWhenActive(t *testing.T) {
|
|
for _, access := range []string{`{{with $alias}}{{.audience}}{{end}}`, `{{index $alias "audience"}}`, `{{index ($alias) "audience"}}`, `{{$alias.audience}}`} {
|
|
for _, active := range []bool{false, true} {
|
|
source := `{{if false}}{{.optional}}{{end}}{{$alias := .}}` + access
|
|
if !active {
|
|
source = `{{if false}}` + source + `{{end}}`
|
|
}
|
|
compiled := compileRequirementTemplate(t, "unknown-scope", source)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
|
}, map[string]string{"audience": "visible"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantKind := domain.RequirementOptional
|
|
if active {
|
|
wantKind = domain.RequirementConditional
|
|
}
|
|
if result.Complete == active || len(result.Variables) != 1 || result.Variables[0].Name != "optional" || result.Variables[0].Kind != wantKind {
|
|
t.Fatalf("active=%v, source=%s, result=%+v", active, source, result)
|
|
}
|
|
if active && (len(result.Issues) == 0 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct) {
|
|
t.Fatalf("active unknown scope lacks a structured issue: %+v", result)
|
|
}
|
|
if !active && len(result.Issues) != 0 {
|
|
t.Fatalf("skipped unknown scope introduced uncertainty: %+v", result)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
|
|
for _, depth := range []int{6, 12} {
|
|
expression := ".selector"
|
|
for level := 0; level < depth; level++ {
|
|
expression = `input (printf "%s" (` + expression + `))`
|
|
}
|
|
source := "{{" + expression + "}}"
|
|
compiled := compileRequirementTemplate(t, "computed-name", source)
|
|
walker := &requirementWalker{
|
|
ctx: context.Background(), vars: map[string]string{"selector": "binding"},
|
|
inputs: make(map[string]*inputEvidence), variables: make(map[string]*variableEvidence),
|
|
issues: make(map[requirementIssueKey]domain.RequirementIssue), complete: true, rootKnown: true,
|
|
}
|
|
if err := walker.walkTemplate(compiled); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A generous source-relative bound detects exponential rescanning, not
|
|
// an exact visit count or a production analysis limit.
|
|
if walker.dependencies > len(source)*8 {
|
|
t.Fatalf("depth=%d: %d accesses for %d bytes of source", depth, walker.dependencies, len(source))
|
|
}
|
|
result := walker.result(domain.RequirementInspection{})
|
|
if result.Complete || len(result.Variables) != 1 || result.Variables[0].Name != "selector" ||
|
|
result.Variables[0].Kind != domain.RequirementConditional || len(result.Issues) == 0 {
|
|
t.Fatalf("nested unsupported names lost discovery metadata: %+v", result)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
|
|
description := "retained metadata"
|
|
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
|
|
snapshot := RequirementSnapshot{
|
|
Inputs: []InputMetadata{
|
|
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
|
|
{Name: "unused", Required: true},
|
|
{Name: "transcript", Required: true},
|
|
},
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", `{{input "transcript"}} {{input "extra"}} {{input "extra"}}`),
|
|
}},
|
|
}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
|
|
t.Fatalf("unexpected result identity or completeness: %+v", result)
|
|
}
|
|
wantNames := []string{"extra", "optional", "transcript", "unused"}
|
|
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantNames) {
|
|
t.Fatalf("input names = %v, want %v", got, wantNames)
|
|
}
|
|
if got := result.Inputs[0]; got.Kind != domain.RequirementRequired || got.Declared || got.ContentType != nil || got.Description != nil {
|
|
t.Fatalf("undeclared literal input was not required independently: %+v", got)
|
|
}
|
|
if got := result.Inputs[1]; got.Kind != domain.RequirementOptional || !got.Declared || got.ContentType == nil || *got.ContentType != "text/plain" ||
|
|
got.Description == nil || *got.Description != description || got.Explanation == "" {
|
|
t.Fatalf("optional declaration metadata was not retained: %+v", got)
|
|
}
|
|
if result.Inputs[2].Kind != domain.RequirementRequired || !result.Inputs[2].Declared || result.Inputs[3].Kind != domain.RequirementRequired {
|
|
t.Fatalf("required declarations or literal use were lost: %+v", result.Inputs)
|
|
}
|
|
*result.Inputs[1].Description = "caller mutation"
|
|
if *snapshot.Inputs[0].Description != description {
|
|
t.Fatal("result metadata pointer aliases the captured snapshot")
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsDistinguishesDirectAndOptionalCallerKeys(t *testing.T) {
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", `{{.missing}} {{$.present}} {{index . "lookup"}}`),
|
|
}},
|
|
}, map[string]string{"present": ""}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Issues) != 0 {
|
|
t.Fatalf("plain root and literal lookup syntax should be complete: %+v", result)
|
|
}
|
|
wantNames := []string{"lookup", "missing", "present"}
|
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
|
t.Fatalf("variable names = %v, want %v", got, wantNames)
|
|
}
|
|
if result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied || result.Variables[0].Explanation == "" {
|
|
t.Errorf("literal index lookup should be optional and absent: %+v", result.Variables[0])
|
|
}
|
|
if result.Variables[1].Kind != domain.RequirementRequired || result.Variables[1].Supplied {
|
|
t.Errorf("missing direct variable should be required but not supplied: %+v", result.Variables[1])
|
|
}
|
|
if result.Variables[2].Kind != domain.RequirementRequired || !result.Variables[2].Supplied {
|
|
t.Errorf("empty supplied direct variable was not detected by presence: %+v", result.Variables[2])
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsSeparatesRootAndLocalTemplateValues(t *testing.T) {
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", `{{$local := "local"}}{{with .context}}{{$local}} {{.child}} {{$.root}}{{end}}`),
|
|
}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantNames := []string{"context", "root"}
|
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
|
|
t.Fatalf("root variable names = %v, want %v", got, wantNames)
|
|
}
|
|
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementConditional || result.Complete {
|
|
t.Fatalf("with scope and unresolved branch should be conservative: %+v", result)
|
|
}
|
|
if len(result.Issues) != 1 || result.Issues[0].Code != domain.RequirementIssueMissingVariable || result.Issues[0].VariableName != "context" {
|
|
t.Fatalf("conditional root use should have one associated limitation: %+v", result.Issues)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsMarksBranchAndUnsupportedScopeReferences(t *testing.T) {
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", `{{if .enabled}}{{input "maybe"}}{{end}}{{range .items}}{{input "inside"}} {{.local}} {{$.root}}{{end}}`),
|
|
}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantInputs := []string{"inside", "maybe"}
|
|
wantVars := []string{"enabled", "items", "root"}
|
|
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantInputs) {
|
|
t.Fatalf("input names = %v, want %v", got, wantInputs)
|
|
}
|
|
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantVars) {
|
|
t.Fatalf("variable names = %v, want %v", got, wantVars)
|
|
}
|
|
if result.Complete || len(result.Issues) < 2 {
|
|
t.Fatalf("branch/range limitations were not reported: %+v", result)
|
|
}
|
|
for _, input := range result.Inputs {
|
|
if input.Kind != domain.RequirementConditional {
|
|
t.Errorf("input %q should be conservative conditional: %+v", input.Name, input)
|
|
}
|
|
}
|
|
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementRequired ||
|
|
result.Variables[2].Kind != domain.RequirementConditional {
|
|
t.Errorf("root and loop-scope variables were misclassified: %+v", result.Variables)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsExcludesUnusedDefinitionsAndBypassedSession(t *testing.T) {
|
|
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}main`)
|
|
session := compileRequirementTemplate(t, "session", `{{input "session_only"}}`)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
|
}, nil, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Inputs) != 0 {
|
|
t.Fatalf("unused definitions or bypassed session were included: %+v", result)
|
|
}
|
|
result, err = AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.Inputs) != 1 || result.Inputs[0].Name != "session_only" || result.Inputs[0].Kind != domain.RequirementRequired {
|
|
t.Fatalf("applicable session template was not visited: %+v", result)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsScansInvokedDefinitionsConservatively(t *testing.T) {
|
|
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}{{define "called"}}{{input "called"}} {{.local}}{{end}}main {{template "called" .}}`)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{message}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Complete || len(result.Inputs) != 1 || result.Inputs[0].Name != "called" ||
|
|
result.Inputs[0].Kind != domain.RequirementConditional || len(result.Variables) != 0 {
|
|
t.Fatalf("called definition was not scanned conservatively: %+v", result)
|
|
}
|
|
if len(result.Issues) < 1 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct {
|
|
t.Fatalf("named-template limitation was not reported: %+v", result.Issues)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
condition string
|
|
vars map[string]string
|
|
wantActive string
|
|
wantSkipped string
|
|
}{
|
|
{name: "string false is truthy", condition: `.flag`, vars: map[string]string{"flag": "false"}, wantActive: "active", wantSkipped: "inactive"},
|
|
{name: "empty direct value is false", condition: `.flag`, vars: map[string]string{"flag": ""}, wantActive: "inactive", wantSkipped: "active"},
|
|
{name: "simple equality", condition: `eq .mode "active"`, vars: map[string]string{"mode": "active"}, wantActive: "active", wantSkipped: "inactive"},
|
|
{name: "simple inequality", condition: `ne .mode "active"`, vars: map[string]string{"mode": "other"}, wantActive: "active", wantSkipped: "inactive"},
|
|
{name: "not direct value", condition: `not .flag`, vars: map[string]string{"flag": ""}, wantActive: "active", wantSkipped: "inactive"},
|
|
{name: "literal true", condition: `true`, wantActive: "active", wantSkipped: "inactive"},
|
|
{name: "literal false", condition: `false`, wantActive: "inactive", wantSkipped: "active"},
|
|
{name: "missing index is empty", condition: `index . "toggle"`, wantActive: "inactive", wantSkipped: "active"},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
|
Role: "user", Content: `{{if ` + test.condition + `}}{{input "active"}}{{else}}{{input "inactive"}}{{end}}`,
|
|
}}}
|
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
|
}}}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Issues) != 0 {
|
|
t.Fatalf("known condition should produce a complete result: %+v", result)
|
|
}
|
|
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
|
for _, input := range result.Inputs {
|
|
inputs[input.Name] = input
|
|
}
|
|
if inputs[test.wantActive].Kind != domain.RequirementRequired || inputs[test.wantSkipped].Kind != domain.RequirementOptional || inputs[test.wantSkipped].Explanation == "" {
|
|
t.Fatalf("selected/skipped inputs were misclassified: %+v", result.Inputs)
|
|
}
|
|
if test.condition == `index . "toggle"` {
|
|
if len(result.Variables) != 1 || result.Variables[0].Name != "toggle" || result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied {
|
|
t.Fatalf("missing index key was not retained as optional: %+v", result.Variables)
|
|
}
|
|
}
|
|
|
|
artifacts := map[string]*domain.Artifact{
|
|
"active": {Name: "active", Body: []byte("active")},
|
|
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
|
}
|
|
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selected := "active"
|
|
if inputs[selected].Kind != domain.RequirementRequired {
|
|
selected = "inactive"
|
|
}
|
|
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != selected {
|
|
t.Fatalf("analysis and renderer disagree: result=%+v rendered=%+v", result, rendered)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsHandlesWithDotAndNestedReachability(t *testing.T) {
|
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
|
Role: "user", Content: `{{with .context}}{{input "inside"}} {{.child}} {{$.root}}{{else}}{{input "empty"}}{{end}}`,
|
|
}}}
|
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
|
}}}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"context": "value", "root": "root"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Inputs) != 2 || len(result.Variables) != 2 {
|
|
t.Fatalf("known with condition should have a complete inventory: %+v", result)
|
|
}
|
|
if result.Inputs[0].Name != "empty" || result.Inputs[0].Kind != domain.RequirementOptional ||
|
|
result.Inputs[1].Name != "inside" || result.Inputs[1].Kind != domain.RequirementRequired {
|
|
t.Fatalf("with body and else reachability is incorrect: %+v", result.Inputs)
|
|
}
|
|
if result.Variables[0].Name != "context" || result.Variables[0].Kind != domain.RequirementRequired ||
|
|
result.Variables[1].Name != "root" || result.Variables[1].Kind != domain.RequirementRequired {
|
|
t.Fatalf("with dot was confused with the root caller map: %+v", result.Variables)
|
|
}
|
|
|
|
nested := compileRequirementTemplate(t, "nested", `{{if .outer}}{{if .inner}}{{input "inner"}}{{else}}{{input "inner_else"}}{{end}}{{else}}{{input "outer_else"}}{{end}}`)
|
|
nestedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{nested}},
|
|
}, map[string]string{"outer": "yes", "inner": "yes"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !nestedResult.Complete || nestedResult.Inputs[0].Name != "inner" || nestedResult.Inputs[0].Kind != domain.RequirementRequired ||
|
|
nestedResult.Inputs[1].Name != "inner_else" || nestedResult.Inputs[1].Kind != domain.RequirementOptional ||
|
|
nestedResult.Inputs[2].Name != "outer_else" || nestedResult.Inputs[2].Kind != domain.RequirementOptional {
|
|
t.Fatalf("known nested conditions lost selected or inactive reachability: %+v", nestedResult)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *testing.T) {
|
|
missing := compileRequirementTemplate(t, "missing", `{{if eq .mode "active"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
|
missingResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{missing}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if missingResult.Complete || len(missingResult.Issues) != 1 || missingResult.Issues[0].Code != domain.RequirementIssueMissingVariable ||
|
|
missingResult.Issues[0].VariableName != "mode" || missingResult.Inputs[0].Kind != domain.RequirementConditional ||
|
|
missingResult.Inputs[1].Kind != domain.RequirementConditional {
|
|
t.Fatalf("missing decision value did not produce conditional dependencies: %+v", missingResult)
|
|
}
|
|
|
|
content := compileRequirementTemplate(t, "content", `{{if input "selector"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
|
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{content}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
contentInputs := make(map[string]domain.InputRequirement, len(contentResult.Inputs))
|
|
for _, input := range contentResult.Inputs {
|
|
contentInputs[input.Name] = input
|
|
}
|
|
if contentResult.Complete || len(contentResult.Issues) != 1 || contentResult.Issues[0].Code != domain.RequirementIssueInputContent ||
|
|
contentResult.Issues[0].InputName != "selector" || contentInputs["selector"].Kind != domain.RequirementRequired ||
|
|
contentInputs["yes"].Kind != domain.RequirementConditional || contentInputs["no"].Kind != domain.RequirementConditional {
|
|
t.Fatalf("content-dependent condition did not preserve known dependencies: %+v", contentResult)
|
|
}
|
|
|
|
textOnly := compileRequirementTemplate(t, "text-only", `{{if .missing}}plain text{{else}}also plain text{{end}}`)
|
|
textResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{textOnly}},
|
|
}, nil, false)
|
|
if err != nil || !textResult.Complete || len(textResult.Issues) != 0 {
|
|
t.Fatalf("unresolved text-only branch should not affect completeness: %+v, %v", textResult, err)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
|
description := "declared for a reason"
|
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
|
Role: "user", Content: `{{if false}}{{input "shared"}} {{.shared_var}} {{input "declared"}}{{input .dynamic}}{{and .left .right}}{{end}}{{input "shared"}} {{.shared_var}}`,
|
|
}}}
|
|
snapshot := RequirementSnapshot{
|
|
Inputs: []InputMetadata{{Name: "declared", Required: true, Description: &description}},
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
|
}},
|
|
}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"shared_var": "present"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Issues) != 0 {
|
|
t.Fatalf("skipped unsupported expressions introduced uncertainty: %+v", result)
|
|
}
|
|
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
|
|
for _, input := range result.Inputs {
|
|
inputs[input.Name] = input
|
|
}
|
|
if len(inputs) != 2 || inputs["shared"].Kind != domain.RequirementRequired || inputs["shared"].Explanation != "" ||
|
|
inputs["declared"].Kind != domain.RequirementRequired || inputs["declared"].Description == nil || *inputs["declared"].Description != description {
|
|
t.Fatalf("active or declared requirements did not override inactivity: %+v", result.Inputs)
|
|
}
|
|
variables := make(map[string]domain.VariableRequirement, len(result.Variables))
|
|
for _, variable := range result.Variables {
|
|
variables[variable.Name] = variable
|
|
}
|
|
for _, name := range []string{"dynamic", "left", "right", "shared_var"} {
|
|
if _, ok := variables[name]; !ok {
|
|
t.Errorf("inactive caller key %q was not inventoried", name)
|
|
}
|
|
}
|
|
if variables["shared_var"].Kind != domain.RequirementRequired || variables["shared_var"].Explanation != "" ||
|
|
variables["dynamic"].Kind != domain.RequirementOptional || variables["dynamic"].Explanation == "" {
|
|
t.Fatalf("inactive caller metadata was not promoted or explained: %+v", result.Variables)
|
|
}
|
|
|
|
allOptional := compileRequirementTemplate(t, "optional", `{{index . "lookup"}}`)
|
|
optionalResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Inputs: []InputMetadata{{Name: "optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{allOptional}},
|
|
}, nil, false)
|
|
if err != nil || !optionalResult.Complete || len(optionalResult.Issues) != 0 || optionalResult.Inputs[0].Kind != domain.RequirementOptional ||
|
|
optionalResult.Variables[0].Kind != domain.RequirementOptional {
|
|
t.Fatalf("all-optional inventory should be complete without artifacts: %+v, %v", optionalResult, err)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
expression string
|
|
vars map[string]string
|
|
kind domain.RequirementKind
|
|
supplied bool
|
|
}{
|
|
{name: "direct dot", expression: `{{input .name}}`, vars: map[string]string{"name": " chosen "}, kind: domain.RequirementRequired, supplied: true},
|
|
{name: "explicit root", expression: `{{input $.name}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
|
{name: "parenthesized direct", expression: `{{input (.name)}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
|
{name: "literal lookup", expression: `{{input (index . "name")}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementOptional, supplied: true},
|
|
{name: "direct pipeline", expression: `{{.name | input}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
|
|
{name: "empty supplied value", expression: `{{input .name}}`, vars: map[string]string{"name": ""}, kind: domain.RequirementRequired, supplied: true},
|
|
{name: "missing lookup has empty zero value", expression: `{{input (index . "name")}}`, vars: nil, kind: domain.RequirementOptional, supplied: false},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{Role: "user", Content: test.expression}}}
|
|
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", test.expression),
|
|
}}}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inputName := "chosen"
|
|
if test.name == "direct dot" {
|
|
inputName = " chosen "
|
|
}
|
|
if test.name == "empty supplied value" || test.name == "missing lookup has empty zero value" {
|
|
inputName = ""
|
|
}
|
|
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != inputName ||
|
|
result.Inputs[0].Kind != domain.RequirementRequired {
|
|
t.Fatalf("simple input name was not resolved exactly: %+v", result)
|
|
}
|
|
if len(result.Variables) != 1 || result.Variables[0].Name != "name" || result.Variables[0].Kind != test.kind || result.Variables[0].Supplied != test.supplied {
|
|
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
|
}
|
|
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
|
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != "resolved" {
|
|
t.Fatalf("analyzed input name differs from renderer: %+v", rendered)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsReportsUnresolvedInputNamesAndPromotesUnknownEffects(t *testing.T) {
|
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
|
Role: "user", Content: `{{input .missing}}`,
|
|
}}}
|
|
result, err := AnalyzeRequirements(context.Background(), definition, RequirementSnapshot{
|
|
Inputs: []InputMetadata{{Name: "optional"}, {Name: "required", Required: true}},
|
|
Templates: CompiledTemplates{Messages: []*template.Template{
|
|
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
|
|
}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Complete || len(result.Issues) != 2 || result.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
|
result.Issues[1].Code != domain.RequirementIssueMissingVariable {
|
|
t.Fatalf("missing dynamic name did not report its limitations: %+v", result)
|
|
}
|
|
if result.Inputs[0].Name != "optional" || result.Inputs[0].Kind != domain.RequirementConditional || result.Inputs[0].Explanation == "" ||
|
|
result.Inputs[1].Name != "required" || result.Inputs[1].Kind != domain.RequirementRequired || len(result.Variables) != 1 ||
|
|
result.Variables[0].Name != "missing" || result.Variables[0].Kind != domain.RequirementRequired {
|
|
t.Fatalf("unknown name effects lost known classifications: %+v", result)
|
|
}
|
|
|
|
computed := compileRequirementTemplate(t, "computed", `{{input (printf "%s" .prefix)}}`)
|
|
computedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Inputs: []InputMetadata{{Name: "declared"}}, Templates: CompiledTemplates{Messages: []*template.Template{computed}},
|
|
}, map[string]string{"prefix": "SENSITIVE-VALUE"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if computedResult.Complete || len(computedResult.Inputs) != 1 || computedResult.Inputs[0].Name != "declared" ||
|
|
computedResult.Inputs[0].Kind != domain.RequirementConditional || len(computedResult.Variables) != 1 ||
|
|
computedResult.Variables[0].Name != "prefix" || computedResult.Variables[0].Kind != domain.RequirementConditional {
|
|
t.Fatalf("computed-name uncertainty did not affect optional entries: %+v", computedResult)
|
|
}
|
|
for _, issue := range computedResult.Issues {
|
|
if strings.Contains(issue.Description, "SENSITIVE-VALUE") || issue.InputName == "SENSITIVE-VALUE" || issue.VariableName == "SENSITIVE-VALUE" {
|
|
t.Fatalf("input-name source value leaked into diagnostics: %+v", issue)
|
|
}
|
|
}
|
|
for _, input := range computedResult.Inputs {
|
|
if strings.Contains(input.Explanation, "SENSITIVE-VALUE") {
|
|
t.Fatalf("input-name source value leaked into an explanation: %+v", input)
|
|
}
|
|
}
|
|
for _, variable := range computedResult.Variables {
|
|
if strings.Contains(variable.Explanation, "SENSITIVE-VALUE") {
|
|
t.Fatalf("input-name source value leaked into an explanation: %+v", variable)
|
|
}
|
|
}
|
|
|
|
for _, expression := range []string{`{{input (input "selector")}}`, `{{input "selector" | input}}`} {
|
|
t.Run(expression, func(t *testing.T) {
|
|
contentName := compileRequirementTemplate(t, "content-name", expression)
|
|
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{contentName}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if contentResult.Complete || len(contentResult.Issues) != 2 ||
|
|
contentResult.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
|
|
contentResult.Issues[1].Code != domain.RequirementIssueInputContent || contentResult.Issues[1].InputName != "selector" || len(contentResult.Inputs) != 1 ||
|
|
contentResult.Inputs[0].Name != "selector" || contentResult.Inputs[0].Kind != domain.RequirementRequired {
|
|
t.Fatalf("input-body-derived name did not preserve the known source dependency: %+v", contentResult)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsDoesNotResolveSkippedComputedInputNames(t *testing.T) {
|
|
compiled := compileRequirementTemplate(t, "inactive-name", `{{if false}}{{input .name}}{{input (printf "%s" .other)}}{{"piped-literal" | input}}{{template "missing" .arg}}{{end}}`)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != "piped-literal" ||
|
|
result.Inputs[0].Kind != domain.RequirementOptional || result.Inputs[0].Explanation == "" || len(result.Variables) != 3 {
|
|
t.Fatalf("skipped computed expressions introduced uncertainty or names: %+v", result)
|
|
}
|
|
for _, variable := range result.Variables {
|
|
if variable.Kind != domain.RequirementOptional || variable.Explanation == "" {
|
|
t.Errorf("skipped variable %q was not retained as inactive optional: %+v", variable.Name, variable)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsPromotesOptionalVariablesWhenRootScopeIsUnknown(t *testing.T) {
|
|
compiled := compileRequirementTemplate(t, "scope", `{{index . "optional"}} {{.required}} {{template "called" .}}{{define "called"}}{{$.hidden}}{{end}}`)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
|
}, map[string]string{"required": "value"}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Complete || len(result.Variables) != 2 || result.Variables[0].Name != "optional" ||
|
|
result.Variables[0].Kind != domain.RequirementConditional || result.Variables[0].Explanation == "" ||
|
|
result.Variables[1].Name != "required" || result.Variables[1].Kind != domain.RequirementRequired {
|
|
t.Fatalf("unknown caller scope did not affect only optional known entries: %+v", result)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsLeavesAssignmentDerivedNamesPartial(t *testing.T) {
|
|
compiled := compileRequirementTemplate(t, "assignment", `{{$alias := .source}}{{input $alias}}`)
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Inputs: []InputMetadata{{Name: "known-optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
|
|
}, nil, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Complete || len(result.Issues) == 0 || len(result.Inputs) != 1 ||
|
|
result.Inputs[0].Name != "known-optional" || result.Inputs[0].Kind != domain.RequirementConditional ||
|
|
len(result.Variables) != 1 || result.Variables[0].Name != "source" || result.Variables[0].Kind != domain.RequirementConditional {
|
|
t.Fatalf("assignment-derived input name was interpreted or lost: %+v", result)
|
|
}
|
|
}
|
|
|
|
func TestAnalyzeRequirementsSupportsEmptyTemplatesAndCancellation(t *testing.T) {
|
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
|
|
Templates: CompiledTemplates{Messages: []*template.Template{compileRequirementTemplate(t, "message", "plain text")}},
|
|
}, nil, false)
|
|
if err != nil || !result.Complete || len(result.Inputs) != 0 || len(result.Variables) != 0 {
|
|
t.Fatalf("input-free template should have a complete empty inventory: %+v, %v", result, err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := AnalyzeRequirements(ctx, &domain.PromptDefinition{}, RequirementSnapshot{}, nil, false); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("canceled context returned %v, want context.Canceled", err)
|
|
}
|
|
}
|
|
|
|
func compileRequirementTemplate(t *testing.T, name, source string) *template.Template {
|
|
t.Helper()
|
|
compiled, err := CompileTemplate(name, source)
|
|
if err != nil {
|
|
t.Fatalf("compile template: %v", err)
|
|
}
|
|
return compiled
|
|
}
|
|
|
|
func requirementInputNames(inputs []domain.InputRequirement) []string {
|
|
names := make([]string, len(inputs))
|
|
for index, input := range inputs {
|
|
names[index] = input.Name
|
|
}
|
|
return names
|
|
}
|
|
|
|
func requirementVariableNames(variables []domain.VariableRequirement) []string {
|
|
names := make([]string, len(variables))
|
|
for index, variable := range variables {
|
|
names[index] = variable.Name
|
|
}
|
|
return names
|
|
}
|
|
|
|
func stringPointerForTest(value string) *string {
|
|
return &value
|
|
}
|