From a93b799236b1b642c78b68ad121599e2d6233dc6 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 11 Aug 2026 22:55:17 +0000 Subject: [PATCH] Preserve exact JSON validation semantics --- engine_test.go | 54 ++++ internal/validate/standard_validator.go | 43 +-- internal/validate/standard_validator_test.go | 268 ++++++++++++++++++- 3 files changed, 347 insertions(+), 18 deletions(-) diff --git a/engine_test.go b/engine_test.go index 6598590..dc90f4c 100644 --- a/engine_test.go +++ b/engine_test.go @@ -2424,6 +2424,60 @@ func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { } } +func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) { + const schema = `{ + "type": "number", + "const": 9007199254740993, + "minimum": 0.123456789012345678901234567890, + "maximum": 1e400, + "multipleOf": 0.0000000000000000001 +}` + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(publicStructuredPromptFS("exact.schema.prompt", "events.schema.json"), "prompts"), + promptkit.WithSchemaFS(fstest.MapFS{ + "schemas/events.schema.json": &fstest.MapFile{Data: []byte(schema)}, + }, "schemas"), + promptkit.WithProfiles(promptkit.Profile{ + ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "exact-model", + }), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: "exact.schema.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if err != nil { + t.Fatalf("prepare structured prompt: %v", err) + } + if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { + t.Fatalf("structured output = %#v, want JSON Schema", prepared.StructuredOutput) + } + document, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) + if !ok { + t.Fatalf("public schema = %#v, want object", prepared.StructuredOutput.JSONSchema.Schema) + } + want := map[string]string{ + "const": "9007199254740993", + "minimum": "0.123456789012345678901234567890", + "maximum": "1e400", + "multipleOf": "0.0000000000000000001", + } + for name, wantNumber := range want { + got, ok := document[name].(json.Number) + if !ok { + t.Fatalf("public schema field %q = %#v, want json.Number", name, document[name]) + } + if got.String() != wantNumber { + t.Fatalf("public schema field %q = %q, want %q", name, got, wantNumber) + } + } +} + func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} engine, err := promptkit.NewEngine(promptkit.Config{ diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go index 7877178..5e0fc8e 100644 --- a/internal/validate/standard_validator.go +++ b/internal/validate/standard_validator.go @@ -1,10 +1,12 @@ package validate import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "io/fs" "net/url" "os" @@ -175,18 +177,17 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d res.IsValid = true return res, nil case domain.ValidationJSON: - _, jsonErr := parseJSON(artifact.Body) - if jsonErr != nil { + if !json.Valid(artifact.Body) { res.Status = domain.ValidationFailed res.IsValid = false - res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} + res.Errors = []string{"invalid JSON"} return res, nil } res.Status = domain.ValidationPassed res.IsValid = true return res, nil case domain.ValidationJSONSchema: - instance, jsonErr := parseJSON(artifact.Body) + instance, jsonErr := decodeJSONValue(artifact.Body) if jsonErr != nil { res.Status = domain.ValidationFailed res.IsValid = false @@ -260,12 +261,22 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str return nil, nil } -func parseJSON(body []byte) (any, error) { - var v any - if err := json.Unmarshal(body, &v); err != nil { +func decodeJSONValue(body []byte) (any, error) { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + + var value any + if err := decoder.Decode(&value); err != nil { return nil, err } - return v, nil + + var trailing any + if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) { + return value, nil + } else if err != nil { + return nil, err + } + return nil, errors.New("multiple JSON values") } func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { @@ -285,8 +296,8 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) } - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { + doc, err := decodeJSONValue(raw) + if err != nil { return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } if err := validateSchemaDialect(doc); err != nil { @@ -356,8 +367,8 @@ func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) } - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { + doc, err := decodeJSONValue(raw) + if err != nil { return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } if err := validateSchemaDialect(doc); err != nil { @@ -490,8 +501,8 @@ func loadJSONSchemaFile(name string) (any, error) { if err != nil { return nil, err } - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { + doc, err := decodeJSONValue(raw) + if err != nil { return nil, err } if err := validateSchemaDialect(doc); err != nil { @@ -538,8 +549,8 @@ func (l fsSchemaLoader) Load(resourceURL string) (any, error) { if err != nil { return nil, err } - var doc any - if err := json.Unmarshal(raw, &doc); err != nil { + doc, err := decodeJSONValue(raw) + if err != nil { return nil, err } if err := validateSchemaDialect(doc); err != nil { diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index 128c8d8..ad80ffc 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -93,6 +93,51 @@ func TestStandardValidatorJSONFailure(t *testing.T) { } } +func TestJSONValidationChecksCompleteSyntaxWithoutChangingArtifact(t *testing.T) { + tests := []struct { + name string + body string + wantValid bool + }{ + {name: "ordinary object", body: `{"count":2,"ok":true}`, wantValid: true}, + {name: "integer at exact float boundary", body: `9007199254740992`, wantValid: true}, + {name: "integer beyond exact float boundary", body: `9007199254740993`, wantValid: true}, + {name: "large exponent", body: `1e400`, wantValid: true}, + {name: "precise decimal", body: `0.123456789012345678901234567890`, wantValid: true}, + {name: "surrounding whitespace", body: " \n [1,2,3] \t", wantValid: true}, + {name: "malformed document", body: `{"count":`, wantValid: false}, + {name: "trailing value", body: `1 2`, wantValid: false}, + } + + validator := NewStandardValidator("") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + body := []byte(tc.body) + before := append([]byte(nil), body...) + artifact := &domain.Artifact{Body: body} + + result, err := validator.Validate(context.Background(), artifact, domain.OutputContract{ + ValidationMode: domain.ValidationJSON, + }) + if err != nil { + t.Fatalf("validate JSON: %v", err) + } + if result.IsValid != tc.wantValid { + t.Fatalf("valid = %v, want %v; result=%+v", result.IsValid, tc.wantValid, result) + } + if tc.wantValid && result.Status != domain.ValidationPassed { + t.Fatalf("status = %q, want %q", result.Status, domain.ValidationPassed) + } + if !tc.wantValid && result.Status != domain.ValidationFailed { + t.Fatalf("status = %q, want %q", result.Status, domain.ValidationFailed) + } + if !reflect.DeepEqual(artifact.Body, before) { + t.Fatalf("artifact body changed: got %q, want %q", artifact.Body, before) + } + }) + } +} + func TestStandardValidatorJSONSchemaSuccess(t *testing.T) { tmp := t.TempDir() schemaPath := filepath.Join(tmp, "schema.json") @@ -629,6 +674,225 @@ func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) { } } +func TestJSONSchemaNumericConstraintsRetainJSONPrecision(t *testing.T) { + tests := []struct { + name string + schema string + instance string + wantValid bool + }{ + { + name: "const distinguishes adjacent large integers", + schema: `{"const":9007199254740993}`, + instance: `9007199254740993`, + wantValid: true, + }, + { + name: "const rejects adjacent large integer", + schema: `{"const":9007199254740993}`, + instance: `9007199254740992`, + wantValid: false, + }, + { + name: "const accepts exponent beyond float range", + schema: `{"const":1e400}`, + instance: `1e400`, + wantValid: true, + }, + { + name: "minimum accepts precise decimal boundary", + schema: `{"type":"number","minimum":0.123456789012345678901234567890}`, + instance: `0.123456789012345678901234567890`, + wantValid: true, + }, + { + name: "minimum rejects lower precise decimal", + schema: `{"type":"number","minimum":0.123456789012345678901234567890}`, + instance: `0.123456789012345678901234567889`, + wantValid: false, + }, + { + name: "maximum distinguishes adjacent large integers", + schema: `{"type":"number","maximum":9007199254740992}`, + instance: `9007199254740993`, + wantValid: false, + }, + { + name: "multiple of accepts exact decimal multiple", + schema: `{"type":"number","multipleOf":0.0000000000000000001}`, + instance: `0.0000000000000000003`, + wantValid: true, + }, + { + name: "multiple of rejects inexact decimal multiple", + schema: `{"type":"number","multipleOf":0.0000000000000000001}`, + instance: `0.00000000000000000031`, + wantValid: false, + }, + { + name: "ordinary number remains supported", + schema: `{"type":"number","minimum":1,"maximum":3}`, + instance: `2`, + wantValid: true, + }, + } + + for _, source := range jsonSchemaValidatorSources() { + t.Run(source.name, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := source.new(t, []byte(tc.schema)) + result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.instance)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("validate JSON Schema instance: %v", err) + } + if result.IsValid != tc.wantValid { + t.Fatalf("valid = %v, want %v; result=%+v", result.IsValid, tc.wantValid, result) + } + }) + } + }) + } +} + +func TestJSONSchemaDecodingRequiresOneCompleteDocument(t *testing.T) { + for _, source := range jsonSchemaValidatorSources() { + t.Run(source.name, func(t *testing.T) { + for _, schema := range []string{`{"type":`, `{} {}`} { + validator := source.new(t, []byte(schema)) + _, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`1`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err == nil { + t.Fatalf("schema %q: expected decoding error", schema) + } + } + + validator := source.new(t, []byte(`{}`)) + for _, instance := range []string{`{"value":`, `1 2`} { + result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(instance)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("instance %q: expected completed validation, got %v", instance, err) + } + if result.Status != domain.ValidationFailed || result.IsValid || len(result.Errors) == 0 { + t.Fatalf("instance %q: expected failed validation, got %+v", instance, result) + } + } + }) + } +} + +func TestPreparedSchemaDocumentsRetainExactNumbers(t *testing.T) { + const schema = `{ + "const": 9007199254740993, + "minimum": 0.123456789012345678901234567890, + "maximum": 1e400, + "multipleOf": 0.0000000000000000001 +}` + want := map[string]string{ + "const": "9007199254740993", + "minimum": "0.123456789012345678901234567890", + "maximum": "1e400", + "multipleOf": "0.0000000000000000001", + } + + for _, source := range jsonSchemaValidatorSources() { + t.Run(source.name, func(t *testing.T) { + validator := source.new(t, []byte(schema)) + preparer, ok := validator.(ValidationPreparer) + if !ok { + t.Fatal("validator does not support validation preparation") + } + prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("prepare validation: %v", err) + } + document, ok := prepared.SchemaDocument().(map[string]any) + if !ok { + t.Fatalf("schema document = %#v, want object", prepared.SchemaDocument()) + } + for name, wantNumber := range want { + got, ok := document[name].(json.Number) + if !ok { + t.Fatalf("schema field %q = %#v, want json.Number", name, document[name]) + } + if got.String() != wantNumber { + t.Fatalf("schema field %q = %q, want %q", name, got, wantNumber) + } + } + }) + } +} + +type jsonSchemaValidatorSource struct { + name string + new func(*testing.T, []byte) Validator +} + +func jsonSchemaValidatorSources() []jsonSchemaValidatorSource { + return []jsonSchemaValidatorSource{ + { + name: "operating system files", + new: func(t *testing.T, schema []byte) Validator { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "schema.json"), schema, 0o644); err != nil { + t.Fatalf("write schema: %v", err) + } + return NewStandardValidator(root) + }, + }, + { + name: "fs.FS", + new: func(t *testing.T, schema []byte) Validator { + t.Helper() + return NewFSValidator(fstest.MapFS{ + "schema.json": &fstest.MapFile{Data: schema}, + }, ".") + }, + }, + } +} + +func BenchmarkJSONValidation(b *testing.B) { + largeArray := []byte(`[` + strings.Repeat(`12345678901234567890,`, 32*1024) + `0]`) + benchmarks := []struct { + name string + body []byte + }{ + {name: "scalar", body: []byte(`1e400`)}, + {name: "object", body: []byte(`{"name":"eris","count":9007199254740993,"enabled":true}`)}, + {name: "large array", body: largeArray}, + } + validator := NewStandardValidator("") + contract := domain.OutputContract{ValidationMode: domain.ValidationJSON} + + for _, benchmark := range benchmarks { + b.Run(benchmark.name, func(b *testing.B) { + artifact := &domain.Artifact{Body: benchmark.body} + b.ReportAllocs() + b.SetBytes(int64(len(benchmark.body))) + b.ResetTimer() + for range b.N { + result, err := validator.Validate(context.Background(), artifact, contract) + if err != nil || !result.IsValid { + b.Fatalf("validate JSON: result=%+v error=%v", result, err) + } + } + }) + } +} + func TestJSONSchemaDialectIsDraft2020(t *testing.T) { tests := []struct { name string @@ -673,8 +937,8 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) { func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) { t.Helper() - var expected any - if err := json.Unmarshal(expectedJSON, &expected); err != nil { + expected, err := decodeJSONValue(expectedJSON) + if err != nil { t.Fatalf("decode expected schema document: %v", err) } if !reflect.DeepEqual(got, expected) {