Files
promptkit/schema_execution_contract_test.go

75 lines
3.6 KiB
Go

package promptkit_test
import (
"context"
"errors"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}}
collection := fstest.MapFS{
"collection/placeholder": {Data: []byte("context")},
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
}
filesystem := configkit.NewFilesystem("collection", collection)
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"},
}}}},
}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"}
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) {
t.Fatalf("configured missing file = %v", err)
}
if len(client.requests) != 0 {
t.Fatal("preparation called model")
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
result, err := engine.Run(context.Background(), base)
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("inline replacement = %+v, %v", result, err)
}
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
t.Fatalf("inline submitted schema = %#v", got)
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
if err := engine.ValidateRequest(context.Background(), base); err != nil {
t.Fatalf("runtime path validation = %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), base)
if err != nil {
t.Fatal(err)
}
result, err = engine.RunPrepared(context.Background(), prepared)
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("root replacement = %+v, %v", result, err)
}
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
t.Fatalf("selected collection root schema = %#v", got)
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("missing replacement selection = %v", err)
}
}
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
engine := newSimpleEngine(t, "hello")
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("missing root = %v", err)
}
}