Verify schema projection and snapshot ownership
This commit is contained in:
@@ -307,6 +307,8 @@ schema plan, without an out-of-band engine schema root.
|
||||
|
||||
## Stage 6 — Verify Public Projections And Ownership
|
||||
|
||||
Status: complete.
|
||||
|
||||
Finish public facade conversion and serialization changes in `convert.go`,
|
||||
`types.go`, `resource_inspection.go`, `json.go`, and prepared notification paths.
|
||||
|
||||
|
||||
@@ -44,6 +44,11 @@ func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
||||
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
|
||||
t.Fatalf("inline submitted schema = %#v", got)
|
||||
}
|
||||
base.Validation.Schema = false
|
||||
result, err = engine.Run(context.Background(), base)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationFailed || client.requests[1].StructuredOutput.JSONSchema.Schema != false {
|
||||
t.Fatalf("false schema = %+v, %v", result, err)
|
||||
}
|
||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
||||
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
||||
t.Fatalf("runtime path validation = %v", err)
|
||||
|
||||
@@ -49,15 +49,22 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
||||
if err := engine.ValidateRequest(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatalf("validate request: %v", err)
|
||||
}
|
||||
inspected, err := engine.InspectPrompt(context.Background(), "prompt")
|
||||
if err != nil || inspected.OutputContract.SchemaFile == nil || *inspected.OutputContract.SchemaFile != "schema.json" {
|
||||
t.Fatalf("schema reference inspection = %+v, %v", inspected, err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil || prepared.Details().StructuredOutput == nil {
|
||||
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||
}
|
||||
if contract := prepared.Details().OutputContract; contract.Schema != nil || contract.SchemaPath != "" {
|
||||
t.Fatalf("source-relative public contract = %+v", contract)
|
||||
}
|
||||
if err := os.Remove(second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed || result.Validation.SchemaPath != "" {
|
||||
t.Fatalf("run = %+v, %v", result, err)
|
||||
}
|
||||
encoded, err := json.Marshal(payload["response_format"])
|
||||
|
||||
138
schema_ownership_contract_test.go
Normal file
138
schema_ownership_contract_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestSchemaProjectionSerialization(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
schema any
|
||||
present bool
|
||||
}{
|
||||
{"absent", nil, false}, {"false", false, true}, {"empty object", map[string]any{}, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
contract := promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: tc.schema}
|
||||
data, err := json.Marshal(promptkit.PreparedRun{OutputContract: contract})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, present := decoded["output_contract"].(map[string]any)["schema"]
|
||||
if present != tc.present || !reflect.DeepEqual(value, tc.schema) {
|
||||
t.Fatalf("schema = %#v, present %v", value, present)
|
||||
}
|
||||
var roundTrip promptkit.PreparedRun
|
||||
if err := json.Unmarshal(data, &roundTrip); err != nil || !reflect.DeepEqual(roundTrip.OutputContract.Schema, tc.schema) {
|
||||
t.Fatalf("decoded = %#v, %v", roundTrip.OutputContract.Schema, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
var old promptkit.OutputContract
|
||||
if err := json.Unmarshal([]byte(`{"format":"json","validation_mode":"json_schema","schema_path":"legacy.json"}`), &old); err != nil || old.Schema != nil || old.SchemaPath != "legacy.json" {
|
||||
t.Fatalf("old contract = %+v, %v", old, err)
|
||||
}
|
||||
file := "nearby.json"
|
||||
for name, codec := range map[string]struct {
|
||||
marshal func(any) ([]byte, error)
|
||||
unmarshal func([]byte, any) error
|
||||
}{"json": {json.Marshal, json.Unmarshal}, "yaml": {yaml.Marshal, yaml.Unmarshal}} {
|
||||
for _, schema := range []any{false, map[string]any{}} {
|
||||
data, err := codec.marshal(promptkit.PromptOutputInspection{Schema: schema, SchemaFile: &file})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value map[string]any
|
||||
if err := codec.unmarshal(data, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := value["schema"]; !ok || !reflect.DeepEqual(got, schema) || value["schema_file"] != file {
|
||||
t.Fatalf("%s inspection = %#v", name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type schemaOwnershipClient struct{ calls int }
|
||||
|
||||
func (client *schemaOwnershipClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.calls++
|
||||
schema := request.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||
if schema["title"] != "captured" {
|
||||
return nil, fmt.Errorf("client saw changed schema: %#v", schema)
|
||||
}
|
||||
schema["title"] = "client mutation"
|
||||
return &promptkit.GenerateResponse{Content: `{"value":3}`}, nil
|
||||
}
|
||||
|
||||
func TestInlineSchemaOwnershipAcrossPublicSnapshots(t *testing.T) {
|
||||
schema := map[string]any{"title": "captured", "type": "object", "properties": map[string]any{"value": map[string]any{"type": "integer"}}}
|
||||
role, content, format, mode := "user", "hello", "json", "json_schema"
|
||||
source := promptkit.PromptSource("prompt", []promptkit.PromptConfig{{ID: "p", Messages: []promptkit.PromptMessageConfig{{Role: &role, Content: &content}}, Output: &promptkit.PromptOutputConfig{Format: &format, ValidationMode: &mode, Schema: schema}}})
|
||||
schema["title"] = "source mutation"
|
||||
endpoint, model := "https://example.invalid/v1", "model"
|
||||
profiles := promptkit.ProfileSource("profile", []promptkit.ProfileConfig{{ID: "profile", Endpoint: &endpoint, Model: &model}})
|
||||
client := &schemaOwnershipClient{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: profiles}, {Source: source}}}}}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "p")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection.OutputContract.Schema.(map[string]any)["title"] = "inspection mutation"
|
||||
request := promptkit.RunRequest{PromptID: "p", ProfileID: "profile"}
|
||||
prepared, err := engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared.OutputContract.Schema.(map[string]any)["title"] = "prepare mutation"
|
||||
var notified bool
|
||||
request.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
|
||||
notified = true
|
||||
details.OutputContract.Schema.(map[string]any)["title"] = "notification mutation"
|
||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "notification mutation"
|
||||
}
|
||||
handle, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
details := handle.Details()
|
||||
details.OutputContract.Schema.(map[string]any)["title"] = "details mutation"
|
||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "details mutation"
|
||||
result, err := engine.RunPrepared(context.Background(), handle)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed || !notified || client.calls != 1 {
|
||||
t.Fatalf("execution = %+v, %v, notified=%v", result, err, notified)
|
||||
}
|
||||
retained := handle.Details()
|
||||
if retained.OutputContract.Schema.(map[string]any)["title"] != "captured" || retained.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "captured" {
|
||||
t.Fatalf("retained schema = %+v", retained)
|
||||
}
|
||||
again, err := engine.InspectPrompt(context.Background(), "p")
|
||||
if err != nil || again.OutputContract.Schema.(map[string]any)["title"] != "captured" {
|
||||
t.Fatalf("inspection = %+v, %v", again, err)
|
||||
}
|
||||
runtimeSchema := map[string]any{"title": "captured", "type": "object"}
|
||||
runtimeRequest := promptkit.RunRequest{PromptID: "p", ProfileID: "profile", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: runtimeSchema}}
|
||||
runtimeHandle, err := engine.PrepareExecution(context.Background(), runtimeRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtimeSchema["title"] = "caller mutation"
|
||||
if result, err := engine.RunPrepared(context.Background(), runtimeHandle); err != nil || result.Validation.Status != promptkit.ValidationPassed || client.calls != 2 {
|
||||
t.Fatalf("runtime schema execution = %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user