From 9bed36282addee323335f2d7aabbbe64528cfd0b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 15:41:13 +0000 Subject: [PATCH] Cover mixed schema sources and captured execution --- docs/roadmap/implementation.md | 2 + schema_collections_contract_test.go | 150 ++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 schema_collections_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 871f720..4693523 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -341,6 +341,8 @@ API requires it, with no resource capability leakage or extra schema work. ## Stage 7 — Verify Mixed Collections And Captured Lifecycles +Status: complete. + Add focused external-package integration coverage using real ConfigKit composition and injected provider/transport boundaries. Prefer a small fixture with distinct risks over a Cartesian product of settings. diff --git a/schema_collections_contract_test.go b/schema_collections_contract_test.go new file mode 100644 index 0000000..a847dc9 --- /dev/null +++ b/schema_collections_contract_test.go @@ -0,0 +1,150 @@ +package promptkit_test + +import ( + "context" + "errors" + "io/fs" + "reflect" + "testing" + "testing/fstest" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type observedSchemaFilesystem struct { + files fstest.MapFS + opens map[string]int +} + +func (source *observedSchemaFilesystem) Open(name string) (fs.File, error) { + source.opens[name]++ + return source.files.Open(name) +} + +func mixedPrompt(id string, output map[string]any) map[string]any { + return map[string]any{"id": id, "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output} +} + +func mixedOutput(field string, value any) map[string]any { + return map[string]any{"format": "json", "validation_mode": "json_schema", field: value} +} + +func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) { + const integer = `{"type":"integer"}` + defaults := &observedSchemaFilesystem{files: fstest.MapFS{ + "embedded/app/schema.json": {Data: []byte(integer)}, + "embedded/app/missing.json": {Data: []byte(integer)}, + "embedded/alpha/schema.json": {Data: []byte(`{"type":"string"}`)}, + "embedded/beta/schema.json": {Data: []byte(integer)}, + "embedded/schema.json": {Data: []byte(integer)}, + }, opens: map[string]int{}} + operator := &observedSchemaFilesystem{files: fstest.MapFS{ + "operator/root/schema.json": {Data: []byte(integer)}, + "operator/schema.json": {Data: []byte(integer)}, + "operator/malformed.json": {Data: []byte(`{`)}, + }, opens: map[string]int{}} + defaultFS := configkit.NewFilesystem("embedded", defaults) + operatorFS := configkit.NewFilesystem("operator", operator) + base := configkit.Input{Source: configkit.Mapping("embedded definitions", map[string]any{ + "schema_root": "app", + "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, + "prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))}, + "collections": map[string]any{ + "alpha": map[string]any{"schema_root": "alpha", "prompts": []any{ + mixedPrompt("base", mixedOutput("schema_file", "schema.json")), + mixedPrompt("replaced", mixedOutput("schema_path", "schema.json")), + mixedPrompt("root", mixedOutput("schema_path", "schema.json")), + mixedPrompt("no-fallback", mixedOutput("schema_path", "missing.json")), + }}, + "beta": map[string]any{"schema_root": "beta", "prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json"))}}, + "gamma": map[string]any{"prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json")), mixedPrompt("inline", mixedOutput("schema", map[string]any{"type": "integer"}))}}, + }, + }), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}} + operatorOutput := mixedOutput("schema_file", "schema.json") + operatorOutput["repair_attempts"] = 1 + overlay := configkit.Input{Source: configkit.Mapping("operator definitions", map[string]any{ + "collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{ + mixedPrompt("replaced", operatorOutput), + map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}}, + map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false}}, + mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")), + }}}, + }), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}} + client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: `"bad"`}, {Content: `42`}, {Content: `42`}}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{base}}}, Main: &overlay}}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"embedded/schema.json", "embedded/app/schema.json", "embedded/beta/schema.json", "operator/schema.json", "operator/root/schema.json", "operator/malformed.json"} { + if defaults.opens[name]+operator.opens[name] != 0 { + t.Fatalf("construction opened %s", name) + } + } + ctx := context.Background() + var expected any + for _, id := range []string{"fallback", "beta/root", "gamma/root", "gamma/inline", "alpha/base", "alpha/inherited", "alpha/root"} { + prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}) + if err != nil { + t.Fatalf("prepare %s: %v", id, err) + } + schema := prepared.StructuredOutput.JSONSchema.Schema + if expected == nil { + expected = schema + } else if !reflect.DeepEqual(expected, schema) { + t.Fatalf("%s document = %#v, want %#v", id, schema, expected) + } + } + if defaults.opens["embedded/schema.json"] != 2 || operator.opens["operator/schema.json"] != 0 { + t.Fatalf("inherited reference did not retain defining filesystem: embedded=%v operator=%v", defaults.opens, operator.opens) + } + for _, id := range []string{"alpha/no-fallback", "alpha/malformed"} { + if _, err := engine.InspectPrompt(ctx, id); err != nil { + t.Fatalf("inspect %s: %v", id, err) + } + if _, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}); !errors.Is(err, promptkit.ErrValidation) { + t.Fatalf("prepare %s = %v", id, err) + } + } + if defaults.opens["embedded/app/missing.json"] != 0 { + t.Fatal("missing collection schema fell back to the application root") + } + request := promptkit.RunRequest{PromptID: "alpha/replaced", ProfileID: "profile"} + prepared, err := engine.PrepareExecution(ctx, request) + if err != nil { + t.Fatal(err) + } + opens := operator.opens["operator/schema.json"] + operator.files["operator/schema.json"] = &fstest.MapFile{Data: []byte(`{"type":"string"}`)} + result, err := engine.RunPrepared(ctx, prepared) + if err != nil || result.Validation.Status != promptkit.ValidationPassed || result.Validation.RepairAttempts != 1 { + t.Fatalf("captured execution = %+v, %v", result, err) + } + if operator.opens["operator/schema.json"] != opens { + t.Fatal("captured execution reopened schema") + } + for _, request := range client.requests[:2] { + if !reflect.DeepEqual(request.StructuredOutput.JSONSchema.Schema, expected) { + t.Fatalf("repair schema = %#v", request.StructuredOutput.JSONSchema.Schema) + } + } + next, err := engine.Prepare(ctx, request) + if err != nil || next.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "string" { + t.Fatalf("later preparation = %+v, %v", next, err) + } + boolean, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "alpha/switched", ProfileID: "profile"}) + if err != nil || boolean.Validation.Status != promptkit.ValidationFailed || client.requests[2].StructuredOutput.JSONSchema.Schema != false { + t.Fatalf("boolean schema = %+v, %v", boolean, err) + } + for _, id := range []string{"beta/root", "gamma/root", "gamma/inline", "alpha/base"} { + client.responses = append(client.responses, &promptkit.GenerateResponse{Content: `42`}) + result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}) + if err != nil || result.Validation.Status != promptkit.ValidationPassed { + t.Fatalf("run %s = %+v, %v", id, result, err) + } + submitted := client.requests[len(client.requests)-1].StructuredOutput.JSONSchema.Schema + if !reflect.DeepEqual(submitted, expected) { + t.Fatalf("%s submitted %#v, want %#v", id, submitted, expected) + } + } +}