Files
promptkit/internal/validate/standard_validator.go

360 lines
9.6 KiB
Go

package validate
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"path"
"strings"
"github.com/santhosh-tekuri/jsonschema/v6"
"go.maximumdirect.net/promptkit/internal/domain"
)
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
const schemaReadChunkSize = 64 * 1024
// Validator prepares source-neutral output validation plans.
type Validator struct{}
func NewValidator() *Validator { return &Validator{} }
type preparedValidation struct {
contract domain.OutputContract
schemaDocument any
schema schemaExecutor
}
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
return validateArtifact(ctx, artifact, p.contract, p.schema)
}
func (p *preparedValidation) SchemaDocument() any {
return p.schemaDocument
}
func (v *Validator) PrepareValidation(ctx context.Context, input Input) (PreparedValidation, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
contract := input.Contract
prepared := &preparedValidation{contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return prepared, nil
}
document := contract.Schema
name := "inline schema"
resourceURL := "promptkit-schema:///inline.json"
if input.File != nil {
if input.File.FS == nil {
return nil, errors.New("schema filesystem is nil")
}
name = input.File.Path
resourceURL = fsSchemaResourceURL(name).String()
raw, err := readSchemaFile(ctx, func() (fs.File, error) { return input.File.FS.Open(name) })
if err != nil {
return nil, fmt.Errorf("failed to read schema file %q: %w", name, err)
}
document, err = decodeJSONValue(ctx, raw)
if err != nil {
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", name, err)
}
}
if document == nil && input.File == nil {
return nil, errors.New("JSON schema document is missing")
}
if err := validateSchemaDialect(document); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", name, err)
}
schema, err := compileSchemaDocument(ctx, resourceURL, document)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", name, err)
}
prepared.schemaDocument, prepared.schema = document, schema
return prepared, nil
}
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, schema schemaExecutor) (domain.ValidationResult, error) {
select {
case <-ctx.Done():
return domain.ValidationResult{}, ctx.Err()
default:
}
res := domain.ValidationResult{
Mode: contract.ValidationMode,
SchemaPath: contract.SchemaPath,
}
if artifact == nil {
return domain.ValidationResult{}, errors.New("artifact is required for validation")
}
switch contract.ValidationMode {
case domain.ValidationNone:
res.Status = domain.ValidationSkipped
res.IsValid = true
return res, nil
case domain.ValidationBasic:
empty := len(bytes.TrimSpace(artifact.Body)) == 0
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
if empty {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"output is empty"}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSON:
valid := json.Valid(artifact.Body)
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
if !valid {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"invalid JSON"}
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
case domain.ValidationJSONSchema:
instance, jsonErr := decodeJSONValue(ctx, artifact.Body)
if jsonErr != nil {
if contextErr := ctx.Err(); contextErr != nil {
return domain.ValidationResult{}, contextErr
}
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
if schema == nil {
return domain.ValidationResult{}, errors.New("prepared JSON schema is unavailable")
}
validationErrors, err := executeJSONSchema(ctx, schema, instance)
if err != nil {
return domain.ValidationResult{}, err
}
if len(validationErrors) > 0 {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = validationErrors
return res, nil
}
res.Status = domain.ValidationPassed
res.IsValid = true
return res, nil
default:
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
}
}
// DecodeInlineSchema converts a caller-supplied value to an owned JSON document.
func DecodeInlineSchema(ctx context.Context, value any) (any, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
encoded, err := json.Marshal(value)
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
if err != nil {
return nil, err
}
return decodeJSONValue(ctx, encoded)
}
func decodeJSONValue(ctx context.Context, body []byte) (any, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.UseNumber()
var value any
decodeErr := decoder.Decode(&value)
if err := ctx.Err(); err != nil {
return nil, err
}
if decodeErr != nil {
return nil, decodeErr
}
var trailing any
trailingErr := decoder.Decode(&trailing)
if err := ctx.Err(); err != nil {
return nil, err
}
if errors.Is(trailingErr, io.EOF) {
return value, nil
} else if trailingErr != nil {
return nil, trailingErr
}
return nil, errors.New("multiple JSON values")
}
func fsSchemaResourceURL(schemaName string) *url.URL {
return &url.URL{
Scheme: "promptkit-schema",
Path: "/" + strings.TrimPrefix(path.Clean(schemaName), "/"),
}
}
type externalSchemaLoader struct{ ctx context.Context }
func (l externalSchemaLoader) Load(resourceURL string) (any, error) {
if err := l.ctx.Err(); err != nil {
return nil, err
}
return nil, fmt.Errorf("external schema resource %q is not supported; use a self-contained schema", resourceURL)
}
func compileSchemaDocument(ctx context.Context, resourceURL string, document any) (*jsonschema.Schema, error) {
root, err := compileSchemaAt(ctx, resourceURL, document)
if err != nil {
return nil, err
}
// File and private-FS retrieval URLs are never available to the provider.
// Compiling again under an unrelated HTTPS base detects reliance on those
// aliases without rewriting references or duplicating URI resolution rules.
if _, err := compileSchemaAt(ctx, "https://promptkit.invalid/submitted-schema", document); err != nil {
return nil, fmt.Errorf("schema is not independent of its source location; use fragment references or explicit schema IDs: %w", err)
}
return root, nil
}
func compileSchemaAt(ctx context.Context, resourceURL string, document any) (*jsonschema.Schema, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
compiler := jsonschema.NewCompiler()
compiler.DefaultDraft(jsonschema.Draft2020)
compiler.UseLoader(externalSchemaLoader{ctx: ctx})
if err := compiler.AddResource(resourceURL, document); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("register schema: %w", err)
}
root, err := compileJSONSchema(ctx, resourceURL, compiler.Compile)
if err != nil {
return nil, err
}
if err := checkSchemaLocations(ctx, compiler, resourceURL, document, "", root); err != nil {
return nil, err
}
return root, nil
}
type schemaExecutor interface {
Validate(instance any) error
}
func compileJSONSchema(ctx context.Context, resourceURL string, compile func(string) (*jsonschema.Schema, error)) (*jsonschema.Schema, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
schema, compileErr := compile(resourceURL)
if err := ctx.Err(); err != nil {
return nil, err
}
return schema, compileErr
}
func executeJSONSchema(ctx context.Context, schema schemaExecutor, instance any) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
validationErr := schema.Validate(instance)
if err := ctx.Err(); err != nil {
return nil, err
}
if validationErr != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", validationErr)}, nil
}
return nil, nil
}
func validateSchemaDialect(doc any) error {
object, ok := doc.(map[string]any)
if !ok {
return nil
}
value, ok := object["$schema"]
if !ok {
return nil
}
dialect, ok := value.(string)
if !ok {
return errors.New("$schema must be a string")
}
if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" {
return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020)
}
return nil
}
func readSchemaFile(ctx context.Context, open func() (fs.File, error)) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
file, openErr := open()
if err := ctx.Err(); err != nil {
if file != nil {
_ = file.Close()
}
return nil, err
}
if openErr != nil {
if file != nil {
_ = file.Close()
}
return nil, openErr
}
if file == nil {
return nil, errors.New("schema source returned a nil file")
}
defer file.Close()
var contents []byte
chunk := make([]byte, schemaReadChunkSize)
for {
if err := ctx.Err(); err != nil {
return nil, err
}
n, readErr := file.Read(chunk)
if n > 0 {
contents = append(contents, chunk[:n]...)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if errors.Is(readErr, io.EOF) {
return contents, nil
}
if readErr != nil {
return nil, readErr
}
if n == 0 {
return nil, io.ErrNoProgress
}
}
}