package promptkit_test import ( "context" "errors" "io/fs" "reflect" "testing" "testing/fstest" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type observedSchemaFilesystem struct { files fstest.MapFS opens map[string]int } func (source *observedSchemaFilesystem) Open(name string) (fs.File, error) { source.opens[name]++ return source.files.Open(name) } func mixedPrompt(id string, output map[string]any) map[string]any { return map[string]any{"id": id, "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output} } func mixedOutput(field string, value any) map[string]any { return map[string]any{"format": "json", "validation_mode": "json_schema", field: value} } func TestMixedSchemaCollectionsAndPreparedLifecycle(t *testing.T) { const integer = `{"type":"integer"}` defaults := &observedSchemaFilesystem{files: fstest.MapFS{ "embedded/app/schema.json": {Data: []byte(integer)}, "embedded/app/missing.json": {Data: []byte(integer)}, "embedded/alpha/schema.json": {Data: []byte(`{"type":"string"}`)}, "embedded/beta/schema.json": {Data: []byte(integer)}, "embedded/schema.json": {Data: []byte(integer)}, }, opens: map[string]int{}} operator := &observedSchemaFilesystem{files: fstest.MapFS{ "operator/root/schema.json": {Data: []byte(integer)}, "operator/schema.json": {Data: []byte(integer)}, "operator/malformed.json": {Data: []byte(`{`)}, }, opens: map[string]int{}} defaultFS := configkit.NewFilesystem("embedded", defaults) operatorFS := configkit.NewFilesystem("operator", operator) base := configkit.Input{Source: configkit.Mapping("embedded definitions", map[string]any{ "schema_root": "app", "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, "prompts": []any{mixedPrompt("fallback", mixedOutput("schema_path", "schema.json"))}, "collections": map[string]any{ "alpha": map[string]any{"schema_root": "alpha", "prompts": []any{ mixedPrompt("base", mixedOutput("schema_file", "schema.json")), mixedPrompt("replaced", mixedOutput("schema_path", "schema.json")), mixedPrompt("root", mixedOutput("schema_path", "schema.json")), mixedPrompt("no-fallback", mixedOutput("schema_path", "missing.json")), }}, "beta": map[string]any{"schema_root": "beta", "prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json"))}}, "gamma": map[string]any{"prompts": []any{mixedPrompt("root", mixedOutput("schema_path", "schema.json")), mixedPrompt("inline", mixedOutput("schema", map[string]any{"type": "integer"}))}}, }, }), Filesystem: &configkit.FilesystemContext{FS: &defaultFS, Root: ".", BaseDir: "embedded"}} operatorOutput := mixedOutput("schema_file", "schema.json") operatorOutput["repair_attempts"] = 1 overlay := configkit.Input{Source: configkit.Mapping("operator definitions", map[string]any{ "collections": map[string]any{"alpha": map[string]any{"schema_root": "root", "prompts": []any{ mixedPrompt("replaced", operatorOutput), map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}}, map[string]any{"id": "switched", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": nil, "schema": false}}, mixedPrompt("malformed", mixedOutput("schema_file", "malformed.json")), }}}, }), Filesystem: &configkit.FilesystemContext{FS: &operatorFS, Root: ".", BaseDir: "operator"}} client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: `"bad"`}, {Content: `42`}, {Content: `42`}}} engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{base}}}, Main: &overlay}}, promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } for _, name := range []string{"embedded/schema.json", "embedded/app/schema.json", "embedded/beta/schema.json", "operator/schema.json", "operator/root/schema.json", "operator/malformed.json"} { if defaults.opens[name]+operator.opens[name] != 0 { t.Fatalf("construction opened %s", name) } } ctx := context.Background() var expected any for _, id := range []string{"fallback", "beta/root", "gamma/root", "gamma/inline", "alpha/base", "alpha/inherited", "alpha/root"} { prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}) if err != nil { t.Fatalf("prepare %s: %v", id, err) } schema := prepared.StructuredOutput.JSONSchema.Schema if expected == nil { expected = schema } else if !reflect.DeepEqual(expected, schema) { t.Fatalf("%s document = %#v, want %#v", id, schema, expected) } } if defaults.opens["embedded/schema.json"] != 2 || operator.opens["operator/schema.json"] != 0 { t.Fatalf("inherited reference did not retain defining filesystem: embedded=%v operator=%v", defaults.opens, operator.opens) } for _, id := range []string{"alpha/no-fallback", "alpha/malformed"} { if _, err := engine.InspectPrompt(ctx, id); err != nil { t.Fatalf("inspect %s: %v", id, err) } if _, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}); !errors.Is(err, promptkit.ErrValidation) { t.Fatalf("prepare %s = %v", id, err) } } if defaults.opens["embedded/app/missing.json"] != 0 { t.Fatal("missing collection schema fell back to the application root") } request := promptkit.RunRequest{PromptID: "alpha/replaced", ProfileID: "profile"} prepared, err := engine.PrepareExecution(ctx, request) if err != nil { t.Fatal(err) } opens := operator.opens["operator/schema.json"] operator.files["operator/schema.json"] = &fstest.MapFile{Data: []byte(`{"type":"string"}`)} result, err := engine.RunPrepared(ctx, prepared) if err != nil || result.Validation.Status != promptkit.ValidationPassed || result.Validation.RepairAttempts != 1 { t.Fatalf("captured execution = %+v, %v", result, err) } if operator.opens["operator/schema.json"] != opens { t.Fatal("captured execution reopened schema") } for _, request := range client.requests[:2] { if !reflect.DeepEqual(request.StructuredOutput.JSONSchema.Schema, expected) { t.Fatalf("repair schema = %#v", request.StructuredOutput.JSONSchema.Schema) } } next, err := engine.Prepare(ctx, request) if err != nil || next.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "string" { t.Fatalf("later preparation = %+v, %v", next, err) } boolean, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "alpha/switched", ProfileID: "profile"}) if err != nil || boolean.Validation.Status != promptkit.ValidationFailed || client.requests[2].StructuredOutput.JSONSchema.Schema != false { t.Fatalf("boolean schema = %+v, %v", boolean, err) } for _, id := range []string{"beta/root", "gamma/root", "gamma/inline", "alpha/base"} { client.responses = append(client.responses, &promptkit.GenerateResponse{Content: `42`}) result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: id, ProfileID: "profile"}) if err != nil || result.Validation.Status != promptkit.ValidationPassed { t.Fatalf("run %s = %+v, %v", id, result, err) } submitted := client.requests[len(client.requests)-1].StructuredOutput.JSONSchema.Schema if !reflect.DeepEqual(submitted, expected) { t.Fatalf("%s submitted %#v, want %#v", id, submitted, expected) } } }