Cover mixed schema sources and captured execution

This commit is contained in:
2026-11-20 15:41:13 +00:00
parent 950d3ccfb2
commit 9bed36282a
2 changed files with 152 additions and 0 deletions

View File

@@ -341,6 +341,8 @@ API requires it, with no resource capability leakage or extra schema work.
## Stage 7 — Verify Mixed Collections And Captured Lifecycles ## Stage 7 — Verify Mixed Collections And Captured Lifecycles
Status: complete.
Add focused external-package integration coverage using real ConfigKit Add focused external-package integration coverage using real ConfigKit
composition and injected provider/transport boundaries. Prefer a small fixture composition and injected provider/transport boundaries. Prefer a small fixture
with distinct risks over a Cartesian product of settings. with distinct risks over a Cartesian product of settings.

View File

@@ -0,0 +1,150 @@
package promptkit_test
import (
"context"
"errors"
"io/fs"
"reflect"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type observedSchemaFilesystem struct {
files fstest.MapFS
opens map[string]int
}
func (source *observedSchemaFilesystem) Open(name string) (fs.File, error) {
source.opens[name]++
return source.files.Open(name)
}
func mixedPrompt(id string, output map[string]any) map[string]any {
return map[string]any{"id": id, "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output}
}
func mixedOutput(field string, value any) map[string]any {
return map[string]any{"format": "json", "validation_mode": "json_schema", field: value}
}
func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) {
const integer = `{"type":"integer"}`
defaults := &observedSchemaFilesystem{files: fstest.MapFS{
"embedded/app/schema.json": {Data: []byte(integer)},
"embedded/app/missing.json": {Data: []byte(integer)},
"embedded/alpha/schema.json": {Data: []byte(`{"type":"string"}`)},
"embedded/beta/schema.json": {Data: []byte(integer)},
"embedded/schema.json": {Data: []byte(integer)},
}, opens: map[string]int{}}
operator := &observedSchemaFilesystem{files: fstest.MapFS{
"operator/root/schema.json": {Data: []byte(integer)},
"operator/schema.json": {Data: []byte(integer)},
"operator/malformed.json": {Data: []byte(`{`)},
}, opens: map[string]int{}}
defaultFS := configkit.NewFilesystem("embedded", defaults)
operatorFS := configkit.NewFilesystem("operator", operator)
base := configkit.Input{Source: configkit.Mapping("embedded definitions", map[string]any{
"schema_root": "app",
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))},
"collections": map[string]any{
"alpha": map[string]any{"schema_root": "alpha", "prompts": []any{
mixedPrompt("base", mixedOutput("schema_file", "schema.json")),
mixedPrompt("replaced", mixedOutput("schema_path", "schema.json")),
mixedPrompt("root", mixedOutput("schema_path", "schema.json")),
mixedPrompt("no-fallback", mixedOutput("schema_path", "missing.json")),
}},
"beta": map[string]any{"schema_root": "beta", "prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json"))}},
"gamma": map[string]any{"prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json")), mixedPrompt("inline", mixedOutput("schema", map[string]any{"type": "integer"}))}},
},
}), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}}
operatorOutput := mixedOutput("schema_file", "schema.json")
operatorOutput["repair_attempts"] = 1
overlay := configkit.Input{Source: configkit.Mapping("operator definitions", map[string]any{
"collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{
mixedPrompt("replaced", operatorOutput),
map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}},
map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false}},
mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")),
}}},
}), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}}
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: `"bad"`}, {Content: `42`}, {Content: `42`}}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{base}}}, Main: &overlay}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"embedded/schema.json", "embedded/app/schema.json", "embedded/beta/schema.json", "operator/schema.json", "operator/root/schema.json", "operator/malformed.json"} {
if defaults.opens[name]+operator.opens[name] != 0 {
t.Fatalf("construction opened %s", name)
}
}
ctx := context.Background()
var expected any
for _, id := range []string{"fallback", "beta/root", "gamma/root", "gamma/inline", "alpha/base", "alpha/inherited", "alpha/root"} {
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"})
if err != nil {
t.Fatalf("prepare %s: %v", id, err)
}
schema := prepared.StructuredOutput.JSONSchema.Schema
if expected == nil {
expected = schema
} else if !reflect.DeepEqual(expected, schema) {
t.Fatalf("%s document = %#v, want %#v", id, schema, expected)
}
}
if defaults.opens["embedded/schema.json"] != 2 || operator.opens["operator/schema.json"] != 0 {
t.Fatalf("inherited reference did not retain defining filesystem: embedded=%v operator=%v", defaults.opens, operator.opens)
}
for _, id := range []string{"alpha/no-fallback", "alpha/malformed"} {
if _, err := engine.InspectPrompt(ctx, id); err != nil {
t.Fatalf("inspect %s: %v", id, err)
}
if _, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}); !errors.Is(err, promptkit.ErrValidation) {
t.Fatalf("prepare %s = %v", id, err)
}
}
if defaults.opens["embedded/app/missing.json"] != 0 {
t.Fatal("missing collection schema fell back to the application root")
}
request := promptkit.RunRequest{PromptID: "alpha/replaced", ProfileID: "profile"}
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
t.Fatal(err)
}
opens := operator.opens["operator/schema.json"]
operator.files["operator/schema.json"] = &fstest.MapFile{Data: []byte(`{"type":"string"}`)}
result, err := engine.RunPrepared(ctx, prepared)
if err != nil || result.Validation.Status != promptkit.ValidationPassed || result.Validation.RepairAttempts != 1 {
t.Fatalf("captured execution = %+v, %v", result, err)
}
if operator.opens["operator/schema.json"] != opens {
t.Fatal("captured execution reopened schema")
}
for _, request := range client.requests[:2] {
if !reflect.DeepEqual(request.StructuredOutput.JSONSchema.Schema, expected) {
t.Fatalf("repair schema = %#v", request.StructuredOutput.JSONSchema.Schema)
}
}
next, err := engine.Prepare(ctx, request)
if err != nil || next.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "string" {
t.Fatalf("later preparation = %+v, %v", next, err)
}
boolean, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "alpha/switched", ProfileID: "profile"})
if err != nil || boolean.Validation.Status != promptkit.ValidationFailed || client.requests[2].StructuredOutput.JSONSchema.Schema != false {
t.Fatalf("boolean schema = %+v, %v", boolean, err)
}
for _, id := range []string{"beta/root", "gamma/root", "gamma/inline", "alpha/base"} {
client.responses = append(client.responses, &promptkit.GenerateResponse{Content: `42`})
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"})
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("run %s = %+v, %v", id, result, err)
}
submitted := client.requests[len(client.requests)-1].StructuredOutput.JSONSchema.Schema
if !reflect.DeepEqual(submitted, expected) {
t.Fatalf("%s submitted %#v, want %#v", id, submitted, expected)
}
}
}