Cover mixed schema sources and captured execution
This commit is contained in:
150
schema_collections_contract_test.go
Normal file
150
schema_collections_contract_test.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type observedSchemaFilesystem struct {
|
||||
files fstest.MapFS
|
||||
opens map[string]int
|
||||
}
|
||||
|
||||
func (source *observedSchemaFilesystem) Open(name string) (fs.File, error) {
|
||||
source.opens[name]++
|
||||
return source.files.Open(name)
|
||||
}
|
||||
|
||||
func mixedPrompt(id string, output map[string]any) map[string]any {
|
||||
return map[string]any{"id": id, "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output}
|
||||
}
|
||||
|
||||
func mixedOutput(field string, value any) map[string]any {
|
||||
return map[string]any{"format": "json", "validation_mode": "json_schema", field: value}
|
||||
}
|
||||
|
||||
func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) {
|
||||
const integer = `{"type":"integer"}`
|
||||
defaults := &observedSchemaFilesystem{files: fstest.MapFS{
|
||||
"embedded/app/schema.json": {Data: []byte(integer)},
|
||||
"embedded/app/missing.json": {Data: []byte(integer)},
|
||||
"embedded/alpha/schema.json": {Data: []byte(`{"type":"string"}`)},
|
||||
"embedded/beta/schema.json": {Data: []byte(integer)},
|
||||
"embedded/schema.json": {Data: []byte(integer)},
|
||||
}, opens: map[string]int{}}
|
||||
operator := &observedSchemaFilesystem{files: fstest.MapFS{
|
||||
"operator/root/schema.json": {Data: []byte(integer)},
|
||||
"operator/schema.json": {Data: []byte(integer)},
|
||||
"operator/malformed.json": {Data: []byte(`{`)},
|
||||
}, opens: map[string]int{}}
|
||||
defaultFS := configkit.NewFilesystem("embedded", defaults)
|
||||
operatorFS := configkit.NewFilesystem("operator", operator)
|
||||
base := configkit.Input{Source: configkit.Mapping("embedded definitions", map[string]any{
|
||||
"schema_root": "app",
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))},
|
||||
"collections": map[string]any{
|
||||
"alpha": map[string]any{"schema_root": "alpha", "prompts": []any{
|
||||
mixedPrompt("base", mixedOutput("schema_file", "schema.json")),
|
||||
mixedPrompt("replaced", mixedOutput("schema_path", "schema.json")),
|
||||
mixedPrompt("root", mixedOutput("schema_path", "schema.json")),
|
||||
mixedPrompt("no-fallback", mixedOutput("schema_path", "missing.json")),
|
||||
}},
|
||||
"beta": map[string]any{"schema_root": "beta", "prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json"))}},
|
||||
"gamma": map[string]any{"prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json")), mixedPrompt("inline", mixedOutput("schema", map[string]any{"type": "integer"}))}},
|
||||
},
|
||||
}), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}}
|
||||
operatorOutput := mixedOutput("schema_file", "schema.json")
|
||||
operatorOutput["repair_attempts"] = 1
|
||||
overlay := configkit.Input{Source: configkit.Mapping("operator definitions", map[string]any{
|
||||
"collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{
|
||||
mixedPrompt("replaced", operatorOutput),
|
||||
map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}},
|
||||
map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false}},
|
||||
mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")),
|
||||
}}},
|
||||
}), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}}
|
||||
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: `"bad"`}, {Content: `42`}, {Content: `42`}}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{base}}}, Main: &overlay}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"embedded/schema.json", "embedded/app/schema.json", "embedded/beta/schema.json", "operator/schema.json", "operator/root/schema.json", "operator/malformed.json"} {
|
||||
if defaults.opens[name]+operator.opens[name] != 0 {
|
||||
t.Fatalf("construction opened %s", name)
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
var expected any
|
||||
for _, id := range []string{"fallback", "beta/root", "gamma/root", "gamma/inline", "alpha/base", "alpha/inherited", "alpha/root"} {
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare %s: %v", id, err)
|
||||
}
|
||||
schema := prepared.StructuredOutput.JSONSchema.Schema
|
||||
if expected == nil {
|
||||
expected = schema
|
||||
} else if !reflect.DeepEqual(expected, schema) {
|
||||
t.Fatalf("%s document = %#v, want %#v", id, schema, expected)
|
||||
}
|
||||
}
|
||||
if defaults.opens["embedded/schema.json"] != 2 || operator.opens["operator/schema.json"] != 0 {
|
||||
t.Fatalf("inherited reference did not retain defining filesystem: embedded=%v operator=%v", defaults.opens, operator.opens)
|
||||
}
|
||||
for _, id := range []string{"alpha/no-fallback", "alpha/malformed"} {
|
||||
if _, err := engine.InspectPrompt(ctx, id); err != nil {
|
||||
t.Fatalf("inspect %s: %v", id, err)
|
||||
}
|
||||
if _, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}); !errors.Is(err, promptkit.ErrValidation) {
|
||||
t.Fatalf("prepare %s = %v", id, err)
|
||||
}
|
||||
}
|
||||
if defaults.opens["embedded/app/missing.json"] != 0 {
|
||||
t.Fatal("missing collection schema fell back to the application root")
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "alpha/replaced", ProfileID: "profile"}
|
||||
prepared, err := engine.PrepareExecution(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opens := operator.opens["operator/schema.json"]
|
||||
operator.files["operator/schema.json"] = &fstest.MapFile{Data: []byte(`{"type":"string"}`)}
|
||||
result, err := engine.RunPrepared(ctx, prepared)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
||||
t.Fatalf("captured execution = %+v, %v", result, err)
|
||||
}
|
||||
if operator.opens["operator/schema.json"] != opens {
|
||||
t.Fatal("captured execution reopened schema")
|
||||
}
|
||||
for _, request := range client.requests[:2] {
|
||||
if !reflect.DeepEqual(request.StructuredOutput.JSONSchema.Schema, expected) {
|
||||
t.Fatalf("repair schema = %#v", request.StructuredOutput.JSONSchema.Schema)
|
||||
}
|
||||
}
|
||||
next, err := engine.Prepare(ctx, request)
|
||||
if err != nil || next.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "string" {
|
||||
t.Fatalf("later preparation = %+v, %v", next, err)
|
||||
}
|
||||
boolean, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "alpha/switched", ProfileID: "profile"})
|
||||
if err != nil || boolean.Validation.Status != promptkit.ValidationFailed || client.requests[2].StructuredOutput.JSONSchema.Schema != false {
|
||||
t.Fatalf("boolean schema = %+v, %v", boolean, err)
|
||||
}
|
||||
for _, id := range []string{"beta/root", "gamma/root", "gamma/inline", "alpha/base"} {
|
||||
client.responses = append(client.responses, &promptkit.GenerateResponse{Content: `42`})
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"})
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||
t.Fatalf("run %s = %+v, %v", id, result, err)
|
||||
}
|
||||
submitted := client.requests[len(client.requests)-1].StructuredOutput.JSONSchema.Schema
|
||||
if !reflect.DeepEqual(submitted, expected) {
|
||||
t.Fatalf("%s submitted %#v, want %#v", id, submitted, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user