139 lines
6.1 KiB
Go
139 lines
6.1 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
func TestSchemaProjectionSerialization(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
schema any
|
|
present bool
|
|
}{
|
|
{"absent", nil, false}, {"false", false, true}, {"empty object", map[string]any{}, true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
contract := promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: tc.schema}
|
|
data, err := json.Marshal(promptkit.PreparedRun{OutputContract: contract})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var decoded map[string]any
|
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value, present := decoded["output_contract"].(map[string]any)["schema"]
|
|
if present != tc.present || !reflect.DeepEqual(value, tc.schema) {
|
|
t.Fatalf("schema = %#v, present %v", value, present)
|
|
}
|
|
var roundTrip promptkit.PreparedRun
|
|
if err := json.Unmarshal(data, &roundTrip); err != nil || !reflect.DeepEqual(roundTrip.OutputContract.Schema, tc.schema) {
|
|
t.Fatalf("decoded = %#v, %v", roundTrip.OutputContract.Schema, err)
|
|
}
|
|
})
|
|
}
|
|
var old promptkit.OutputContract
|
|
if err := json.Unmarshal([]byte(`{"format":"json","validation_mode":"json_schema","schema_path":"legacy.json"}`), &old); err != nil || old.Schema != nil || old.SchemaPath != "legacy.json" {
|
|
t.Fatalf("old contract = %+v, %v", old, err)
|
|
}
|
|
file := "nearby.json"
|
|
for name, codec := range map[string]struct {
|
|
marshal func(any) ([]byte, error)
|
|
unmarshal func([]byte, any) error
|
|
}{"json": {json.Marshal, json.Unmarshal}, "yaml": {yaml.Marshal, yaml.Unmarshal}} {
|
|
for _, schema := range []any{false, map[string]any{}} {
|
|
data, err := codec.marshal(promptkit.PromptOutputInspection{Schema: schema, SchemaFile: &file})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var value map[string]any
|
|
if err := codec.unmarshal(data, &value); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, ok := value["schema"]; !ok || !reflect.DeepEqual(got, schema) || value["schema_file"] != file {
|
|
t.Fatalf("%s inspection = %#v", name, value)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
type schemaOwnershipClient struct{ calls int }
|
|
|
|
func (client *schemaOwnershipClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
client.calls++
|
|
schema := request.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
|
if schema["title"] != "captured" {
|
|
return nil, fmt.Errorf("client saw changed schema: %#v", schema)
|
|
}
|
|
schema["title"] = "client mutation"
|
|
return &promptkit.GenerateResponse{Content: `{"value":3}`}, nil
|
|
}
|
|
|
|
func TestInlineSchemaOwnershipAcrossPublicSnapshots(t *testing.T) {
|
|
schema := map[string]any{"title": "captured", "type": "object", "properties": map[string]any{"value": map[string]any{"type": "integer"}}}
|
|
role, content, format, mode := "user", "hello", "json", "json_schema"
|
|
source := promptkit.PromptSource("prompt", []promptkit.PromptConfig{{ID: "p", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &mode, Schema: schema}}})
|
|
schema["title"] = "source mutation"
|
|
endpoint, model := "https://example.invalid/v1", "model"
|
|
profiles := promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "profile", Endpoint: &endpoint, Model: &model}})
|
|
client := &schemaOwnershipClient{}
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: profiles}, {Source: source}}}}}}, promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inspection, err := engine.InspectPrompt(context.Background(), "p")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inspection.OutputContract.Schema.(map[string]any)["title"] = "inspection mutation"
|
|
request := promptkit.RunRequest{PromptID: "p", ProfileID: "profile"}
|
|
prepared, err := engine.Prepare(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared.OutputContract.Schema.(map[string]any)["title"] = "prepare mutation"
|
|
var notified bool
|
|
request.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
|
|
notified = true
|
|
details.OutputContract.Schema.(map[string]any)["title"] = "notification mutation"
|
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "notification mutation"
|
|
}
|
|
handle, err := engine.PrepareExecution(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
details := handle.Details()
|
|
details.OutputContract.Schema.(map[string]any)["title"] = "details mutation"
|
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "details mutation"
|
|
result, err := engine.RunPrepared(context.Background(), handle)
|
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed || !notified || client.calls != 1 {
|
|
t.Fatalf("execution = %+v, %v, notified=%v", result, err, notified)
|
|
}
|
|
retained := handle.Details()
|
|
if retained.OutputContract.Schema.(map[string]any)["title"] != "captured" || retained.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "captured" {
|
|
t.Fatalf("retained schema = %+v", retained)
|
|
}
|
|
again, err := engine.InspectPrompt(context.Background(), "p")
|
|
if err != nil || again.OutputContract.Schema.(map[string]any)["title"] != "captured" {
|
|
t.Fatalf("inspection = %+v, %v", again, err)
|
|
}
|
|
runtimeSchema := map[string]any{"title": "captured", "type": "object"}
|
|
runtimeRequest := promptkit.RunRequest{PromptID: "p", ProfileID: "profile", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: runtimeSchema}}
|
|
runtimeHandle, err := engine.PrepareExecution(context.Background(), runtimeRequest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtimeSchema["title"] = "caller mutation"
|
|
if result, err := engine.RunPrepared(context.Background(), runtimeHandle); err != nil || result.Validation.Status != promptkit.ValidationPassed || client.calls != 2 {
|
|
t.Fatalf("runtime schema execution = %+v, %v", result, err)
|
|
}
|
|
}
|