Files
promptkit/schema_ownership_contract_test.go

139 lines
6.1 KiB
Go

package promptkit_test
import (
"context"
"encoding/json"
"fmt"
"reflect"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"go.yaml.in/yaml/v4"
)
func TestSchemaProjectionSerialization(t *testing.T) {
for _, tc := range []struct {
name string
schema any
present bool
}{
{"absent", nil, false}, {"false", false, true}, {"empty object", map[string]any{}, true},
} {
t.Run(tc.name, func(t *testing.T) {
contract := promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: tc.schema}
data, err := json.Marshal(promptkit.PreparedRun{OutputContract: contract})
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatal(err)
}
value, present := decoded["output_contract"].(map[string]any)["schema"]
if present != tc.present || !reflect.DeepEqual(value, tc.schema) {
t.Fatalf("schema = %#v, present %v", value, present)
}
var roundTrip promptkit.PreparedRun
if err := json.Unmarshal(data, &roundTrip); err != nil || !reflect.DeepEqual(roundTrip.OutputContract.Schema, tc.schema) {
t.Fatalf("decoded = %#v, %v", roundTrip.OutputContract.Schema, err)
}
})
}
var old promptkit.OutputContract
if err := json.Unmarshal([]byte(`{"format":"json","validation_mode":"json_schema","schema_path":"legacy.json"}`), &old); err != nil || old.Schema != nil || old.SchemaPath != "legacy.json" {
t.Fatalf("old contract = %+v, %v", old, err)
}
file := "nearby.json"
for name, codec := range map[string]struct {
marshal func(any) ([]byte, error)
unmarshal func([]byte, any) error
}{"json": {json.Marshal, json.Unmarshal}, "yaml": {yaml.Marshal, yaml.Unmarshal}} {
for _, schema := range []any{false, map[string]any{}} {
data, err := codec.marshal(promptkit.PromptOutputInspection{Schema: schema, SchemaFile: &file})
if err != nil {
t.Fatal(err)
}
var value map[string]any
if err := codec.unmarshal(data, &value); err != nil {
t.Fatal(err)
}
if got, ok := value["schema"]; !ok || !reflect.DeepEqual(got, schema) || value["schema_file"] != file {
t.Fatalf("%s inspection = %#v", name, value)
}
}
}
}
type schemaOwnershipClient struct{ calls int }
func (client *schemaOwnershipClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.calls++
schema := request.StructuredOutput.JSONSchema.Schema.(map[string]any)
if schema["title"] != "captured" {
return nil, fmt.Errorf("client saw changed schema: %#v", schema)
}
schema["title"] = "client mutation"
return &promptkit.GenerateResponse{Content: `{"value":3}`}, nil
}
func TestInlineSchemaOwnershipAcrossPublicSnapshots(t *testing.T) {
schema := map[string]any{"title": "captured", "type": "object", "properties": map[string]any{"value": map[string]any{"type": "integer"}}}
role, content, format, mode := "user", "hello", "json", "json_schema"
source := promptkit.PromptSource("prompt", []promptkit.PromptConfig{{ID: "p", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &mode, Schema: schema}}})
schema["title"] = "source mutation"
endpoint, model := "https://example.invalid/v1", "model"
profiles := promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "profile", Endpoint: &endpoint, Model: &model}})
client := &schemaOwnershipClient{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: profiles}, {Source: source}}}}}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectPrompt(context.Background(), "p")
if err != nil {
t.Fatal(err)
}
inspection.OutputContract.Schema.(map[string]any)["title"] = "inspection mutation"
request := promptkit.RunRequest{PromptID: "p", ProfileID: "profile"}
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatal(err)
}
prepared.OutputContract.Schema.(map[string]any)["title"] = "prepare mutation"
var notified bool
request.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
notified = true
details.OutputContract.Schema.(map[string]any)["title"] = "notification mutation"
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "notification mutation"
}
handle, err := engine.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatal(err)
}
details := handle.Details()
details.OutputContract.Schema.(map[string]any)["title"] = "details mutation"
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "details mutation"
result, err := engine.RunPrepared(context.Background(), handle)
if err != nil || result.Validation.Status != promptkit.ValidationPassed || !notified || client.calls != 1 {
t.Fatalf("execution = %+v, %v, notified=%v", result, err, notified)
}
retained := handle.Details()
if retained.OutputContract.Schema.(map[string]any)["title"] != "captured" || retained.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "captured" {
t.Fatalf("retained schema = %+v", retained)
}
again, err := engine.InspectPrompt(context.Background(), "p")
if err != nil || again.OutputContract.Schema.(map[string]any)["title"] != "captured" {
t.Fatalf("inspection = %+v, %v", again, err)
}
runtimeSchema := map[string]any{"title": "captured", "type": "object"}
runtimeRequest := promptkit.RunRequest{PromptID: "p", ProfileID: "profile", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: runtimeSchema}}
runtimeHandle, err := engine.PrepareExecution(context.Background(), runtimeRequest)
if err != nil {
t.Fatal(err)
}
runtimeSchema["title"] = "caller mutation"
if result, err := engine.RunPrepared(context.Background(), runtimeHandle); err != nil || result.Validation.Status != promptkit.ValidationPassed || client.calls != 2 {
t.Fatalf("runtime schema execution = %+v, %v", result, err)
}
}