Expose prompt inspection through the engine
This commit is contained in:
@@ -174,6 +174,223 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(fstest.MapFS{
|
||||
"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
|
||||
version: "1.0.0"
|
||||
messages:
|
||||
- role: user
|
||||
content: old report
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
|
||||
version: "2.0.0"
|
||||
default_profile: missing-profile
|
||||
inputs:
|
||||
- name: location
|
||||
required: true
|
||||
content_type: text/plain
|
||||
description: Forecast location.
|
||||
- name: units
|
||||
content_type: text/plain
|
||||
description: Unit preference.
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/report.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schemas/report.json
|
||||
`)},
|
||||
"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
|
||||
}, "."),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||
}
|
||||
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect prompt: %v", err)
|
||||
}
|
||||
if inspection.PromptID != "report" ||
|
||||
inspection.PromptVersion != "2.0.0" ||
|
||||
inspection.PromptHash == "" ||
|
||||
inspection.DefaultProfileID != "missing-profile" ||
|
||||
inspection.OutputContract != (promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSONSchema,
|
||||
SchemaPath: "schemas/report.json",
|
||||
}) {
|
||||
t.Fatalf("unexpected inspection metadata: %#v", inspection)
|
||||
}
|
||||
wantInputs := []promptkit.PromptInputDefinition{
|
||||
{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
|
||||
{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
|
||||
}
|
||||
if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
|
||||
t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
|
||||
newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
|
||||
if err != nil {
|
||||
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
validSource := fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||
version: "1"
|
||||
messages:
|
||||
- role: user
|
||||
content: body
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}
|
||||
|
||||
var nilEngine *promptkit.Engine
|
||||
if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
|
||||
}
|
||||
|
||||
valid := newEngine(t, validSource)
|
||||
if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
|
||||
}
|
||||
if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
|
||||
}
|
||||
|
||||
ambiguous := newEngine(t, fstest.MapFS{
|
||||
"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||
version: "1"
|
||||
messages:
|
||||
- role: user
|
||||
content: first
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||
version: "2"
|
||||
messages:
|
||||
- role: user
|
||||
content: second
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
})
|
||||
if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
|
||||
}
|
||||
|
||||
for name, source := range map[string]fstest.MapFS{
|
||||
"malformed definition": {
|
||||
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
|
||||
},
|
||||
"missing content file": {
|
||||
"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
|
||||
version: "1"
|
||||
messages:
|
||||
- role: user
|
||||
content_file: missing.md
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrPromptLoad) {
|
||||
t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
countingFS := &inspectionCountingFS{}
|
||||
canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
|
||||
if err != nil {
|
||||
t.Fatalf("construct canceled prompt-inspection engine: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
|
||||
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
inputs:
|
||||
- name: subject
|
||||
content_type: text/plain
|
||||
description: Summary subject.
|
||||
messages:
|
||||
- role: user
|
||||
content: summarize
|
||||
output:
|
||||
format: markdown
|
||||
validation_mode: basic
|
||||
`)},
|
||||
}, "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct prompt-inspection engine: %v", err)
|
||||
}
|
||||
|
||||
first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
|
||||
if err != nil {
|
||||
t.Fatalf("first inspection: %v", err)
|
||||
}
|
||||
first.Inputs[0].Name = "changed"
|
||||
first.OutputContract.SchemaPath = "changed.json"
|
||||
|
||||
second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
|
||||
if err != nil {
|
||||
t.Fatalf("second inspection: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare after inspection mutation: %v", err)
|
||||
}
|
||||
if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
|
||||
prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
|
||||
t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
||||
payload, err := json.Marshal(promptkit.PreparedRun{})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user