360 lines
9.6 KiB
Go
360 lines
9.6 KiB
Go
package validate
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"net/url"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
|
"go.maximumdirect.net/promptkit/internal/domain"
|
|
)
|
|
|
|
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
|
|
|
|
const schemaReadChunkSize = 64 * 1024
|
|
|
|
// Validator prepares source-neutral output validation plans.
|
|
type Validator struct{}
|
|
|
|
func NewValidator() *Validator { return &Validator{} }
|
|
|
|
type preparedValidation struct {
|
|
contract domain.OutputContract
|
|
schemaDocument any
|
|
schema schemaExecutor
|
|
}
|
|
|
|
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
|
return validateArtifact(ctx, artifact, p.contract, p.schema)
|
|
}
|
|
|
|
func (p *preparedValidation) SchemaDocument() any {
|
|
return p.schemaDocument
|
|
}
|
|
|
|
func (v *Validator) PrepareValidation(ctx context.Context, input Input) (PreparedValidation, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
contract := input.Contract
|
|
prepared := &preparedValidation{contract: contract}
|
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
|
return prepared, nil
|
|
}
|
|
document := contract.Schema
|
|
name := "inline schema"
|
|
resourceURL := "promptkit-schema:///inline.json"
|
|
if input.File != nil {
|
|
if input.File.FS == nil {
|
|
return nil, errors.New("schema filesystem is nil")
|
|
}
|
|
name = input.File.Path
|
|
resourceURL = fsSchemaResourceURL(name).String()
|
|
raw, err := readSchemaFile(ctx, func() (fs.File, error) { return input.File.FS.Open(name) })
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read schema file %q: %w", name, err)
|
|
}
|
|
document, err = decodeJSONValue(ctx, raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", name, err)
|
|
}
|
|
}
|
|
if document == nil && input.File == nil {
|
|
return nil, errors.New("JSON schema document is missing")
|
|
}
|
|
if err := validateSchemaDialect(document); err != nil {
|
|
if contextErr := ctx.Err(); contextErr != nil {
|
|
return nil, contextErr
|
|
}
|
|
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", name, err)
|
|
}
|
|
schema, err := compileSchemaDocument(ctx, resourceURL, document)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", name, err)
|
|
}
|
|
prepared.schemaDocument, prepared.schema = document, schema
|
|
return prepared, nil
|
|
}
|
|
|
|
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, schema schemaExecutor) (domain.ValidationResult, error) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return domain.ValidationResult{}, ctx.Err()
|
|
default:
|
|
}
|
|
|
|
res := domain.ValidationResult{
|
|
Mode: contract.ValidationMode,
|
|
SchemaPath: contract.SchemaPath,
|
|
}
|
|
|
|
if artifact == nil {
|
|
return domain.ValidationResult{}, errors.New("artifact is required for validation")
|
|
}
|
|
|
|
switch contract.ValidationMode {
|
|
case domain.ValidationNone:
|
|
res.Status = domain.ValidationSkipped
|
|
res.IsValid = true
|
|
return res, nil
|
|
case domain.ValidationBasic:
|
|
empty := len(bytes.TrimSpace(artifact.Body)) == 0
|
|
if err := ctx.Err(); err != nil {
|
|
return domain.ValidationResult{}, err
|
|
}
|
|
if empty {
|
|
res.Status = domain.ValidationFailed
|
|
res.IsValid = false
|
|
res.Errors = []string{"output is empty"}
|
|
return res, nil
|
|
}
|
|
res.Status = domain.ValidationPassed
|
|
res.IsValid = true
|
|
return res, nil
|
|
case domain.ValidationJSON:
|
|
valid := json.Valid(artifact.Body)
|
|
if err := ctx.Err(); err != nil {
|
|
return domain.ValidationResult{}, err
|
|
}
|
|
if !valid {
|
|
res.Status = domain.ValidationFailed
|
|
res.IsValid = false
|
|
res.Errors = []string{"invalid JSON"}
|
|
return res, nil
|
|
}
|
|
res.Status = domain.ValidationPassed
|
|
res.IsValid = true
|
|
return res, nil
|
|
case domain.ValidationJSONSchema:
|
|
instance, jsonErr := decodeJSONValue(ctx, artifact.Body)
|
|
if jsonErr != nil {
|
|
if contextErr := ctx.Err(); contextErr != nil {
|
|
return domain.ValidationResult{}, contextErr
|
|
}
|
|
res.Status = domain.ValidationFailed
|
|
res.IsValid = false
|
|
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
|
|
return res, nil
|
|
}
|
|
|
|
if schema == nil {
|
|
return domain.ValidationResult{}, errors.New("prepared JSON schema is unavailable")
|
|
}
|
|
validationErrors, err := executeJSONSchema(ctx, schema, instance)
|
|
if err != nil {
|
|
return domain.ValidationResult{}, err
|
|
}
|
|
if len(validationErrors) > 0 {
|
|
res.Status = domain.ValidationFailed
|
|
res.IsValid = false
|
|
res.Errors = validationErrors
|
|
return res, nil
|
|
}
|
|
|
|
res.Status = domain.ValidationPassed
|
|
res.IsValid = true
|
|
return res, nil
|
|
default:
|
|
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
|
|
}
|
|
}
|
|
|
|
// DecodeInlineSchema converts a caller-supplied value to an owned JSON document.
|
|
func DecodeInlineSchema(ctx context.Context, value any) (any, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
encoded, err := json.Marshal(value)
|
|
if contextErr := ctx.Err(); contextErr != nil {
|
|
return nil, contextErr
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeJSONValue(ctx, encoded)
|
|
}
|
|
|
|
func decodeJSONValue(ctx context.Context, body []byte) (any, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
|
decoder.UseNumber()
|
|
|
|
var value any
|
|
decodeErr := decoder.Decode(&value)
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if decodeErr != nil {
|
|
return nil, decodeErr
|
|
}
|
|
|
|
var trailing any
|
|
trailingErr := decoder.Decode(&trailing)
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if errors.Is(trailingErr, io.EOF) {
|
|
return value, nil
|
|
} else if trailingErr != nil {
|
|
return nil, trailingErr
|
|
}
|
|
return nil, errors.New("multiple JSON values")
|
|
}
|
|
|
|
func fsSchemaResourceURL(schemaName string) *url.URL {
|
|
return &url.URL{
|
|
Scheme: "promptkit-schema",
|
|
Path: "/" + strings.TrimPrefix(path.Clean(schemaName), "/"),
|
|
}
|
|
}
|
|
|
|
type externalSchemaLoader struct{ ctx context.Context }
|
|
|
|
func (l externalSchemaLoader) Load(resourceURL string) (any, error) {
|
|
if err := l.ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, fmt.Errorf("external schema resource %q is not supported; use a self-contained schema", resourceURL)
|
|
}
|
|
|
|
func compileSchemaDocument(ctx context.Context, resourceURL string, document any) (*jsonschema.Schema, error) {
|
|
root, err := compileSchemaAt(ctx, resourceURL, document)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// File and private-FS retrieval URLs are never available to the provider.
|
|
// Compiling again under an unrelated HTTPS base detects reliance on those
|
|
// aliases without rewriting references or duplicating URI resolution rules.
|
|
if _, err := compileSchemaAt(ctx, "https://promptkit.invalid/submitted-schema", document); err != nil {
|
|
return nil, fmt.Errorf("schema is not independent of its source location; use fragment references or explicit schema IDs: %w", err)
|
|
}
|
|
return root, nil
|
|
}
|
|
|
|
func compileSchemaAt(ctx context.Context, resourceURL string, document any) (*jsonschema.Schema, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
compiler := jsonschema.NewCompiler()
|
|
compiler.DefaultDraft(jsonschema.Draft2020)
|
|
compiler.UseLoader(externalSchemaLoader{ctx: ctx})
|
|
if err := compiler.AddResource(resourceURL, document); err != nil {
|
|
if contextErr := ctx.Err(); contextErr != nil {
|
|
return nil, contextErr
|
|
}
|
|
return nil, fmt.Errorf("register schema: %w", err)
|
|
}
|
|
root, err := compileJSONSchema(ctx, resourceURL, compiler.Compile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := checkSchemaLocations(ctx, compiler, resourceURL, document, "", root); err != nil {
|
|
return nil, err
|
|
}
|
|
return root, nil
|
|
}
|
|
|
|
type schemaExecutor interface {
|
|
Validate(instance any) error
|
|
}
|
|
|
|
func compileJSONSchema(ctx context.Context, resourceURL string, compile func(string) (*jsonschema.Schema, error)) (*jsonschema.Schema, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
schema, compileErr := compile(resourceURL)
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return schema, compileErr
|
|
}
|
|
|
|
func executeJSONSchema(ctx context.Context, schema schemaExecutor, instance any) ([]string, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
validationErr := schema.Validate(instance)
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if validationErr != nil {
|
|
return []string{fmt.Sprintf("json schema validation failed: %v", validationErr)}, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func validateSchemaDialect(doc any) error {
|
|
object, ok := doc.(map[string]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
value, ok := object["$schema"]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
dialect, ok := value.(string)
|
|
if !ok {
|
|
return errors.New("$schema must be a string")
|
|
}
|
|
if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" {
|
|
return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func readSchemaFile(ctx context.Context, open func() (fs.File, error)) ([]byte, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
file, openErr := open()
|
|
if err := ctx.Err(); err != nil {
|
|
if file != nil {
|
|
_ = file.Close()
|
|
}
|
|
return nil, err
|
|
}
|
|
if openErr != nil {
|
|
if file != nil {
|
|
_ = file.Close()
|
|
}
|
|
return nil, openErr
|
|
}
|
|
if file == nil {
|
|
return nil, errors.New("schema source returned a nil file")
|
|
}
|
|
defer file.Close()
|
|
|
|
var contents []byte
|
|
chunk := make([]byte, schemaReadChunkSize)
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
n, readErr := file.Read(chunk)
|
|
if n > 0 {
|
|
contents = append(contents, chunk[:n]...)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if errors.Is(readErr, io.EOF) {
|
|
return contents, nil
|
|
}
|
|
if readErr != nil {
|
|
return nil, readErr
|
|
}
|
|
if n == 0 {
|
|
return nil, io.ErrNoProgress
|
|
}
|
|
}
|
|
}
|