80 lines
3.9 KiB
Go
80 lines
3.9 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
|
app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}}
|
|
collection := fstest.MapFS{
|
|
"collection/placeholder": {Data: []byte("context")},
|
|
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
|
|
}
|
|
filesystem := configkit.NewFilesystem("collection", collection)
|
|
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
|
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
|
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
|
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"},
|
|
}}}},
|
|
}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}}
|
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}}
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"}
|
|
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) {
|
|
t.Fatalf("configured missing file = %v", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatal("preparation called model")
|
|
}
|
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
|
|
result, err := engine.Run(context.Background(), base)
|
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
|
t.Fatalf("inline replacement = %+v, %v", result, err)
|
|
}
|
|
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
|
|
t.Fatalf("inline submitted schema = %#v", got)
|
|
}
|
|
base.Validation.Schema = false
|
|
result, err = engine.Run(context.Background(), base)
|
|
if err != nil || result.Validation.Status != promptkit.ValidationFailed || client.requests[1].StructuredOutput.JSONSchema.Schema != false {
|
|
t.Fatalf("false schema = %+v, %v", result, err)
|
|
}
|
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
|
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
|
t.Fatalf("runtime path validation = %v", err)
|
|
}
|
|
prepared, err := engine.PrepareExecution(context.Background(), base)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err = engine.RunPrepared(context.Background(), prepared)
|
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
|
t.Fatalf("root replacement = %+v, %v", result, err)
|
|
}
|
|
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
|
|
t.Fatalf("selected collection root schema = %#v", got)
|
|
}
|
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
|
|
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
t.Fatalf("missing replacement selection = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
|
|
engine := newSimpleEngine(t, "hello")
|
|
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
|
|
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
t.Fatalf("missing root = %v", err)
|
|
}
|
|
}
|