eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make validation cancellation authoritative

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-11 23:14:01 +00:00
parent 20d3e3b5ee
commit e43350fd0d
7 changed files with 689 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

300
internal/validate/standard_validator.go
View File

@@ -22,6 +22,8 @@ import (
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
const schemaReadChunkSize = 64 * 1024
// StandardValidator provides basic, JSON, and JSON Schema output validation.
type StandardValidator struct {
schemaBaseDir string
@@ -51,7 +53,7 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
type preparedValidation struct {
contract domain.OutputContract
schemaDocument any
schema *jsonschema.Schema
schema schemaExecutor
}
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
@@ -62,14 +64,11 @@ func (p *preparedValidation) SchemaDocument() any {
return p.schemaDocument
}
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
func (p *preparedValidation) validateJSONSchema(ctx context.Context, instance any, _ string) ([]string, error) {
if p.schema == nil {
return nil, errors.New("prepared JSON schema is unavailable")
}
if err := p.schema.Validate(instance); err != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
}
return nil, nil
return executeJSONSchema(ctx, p.schema, instance)
}
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
@@ -82,11 +81,11 @@ func (v *StandardValidator) PrepareValidation(ctx context.Context, contract doma
return prepared, nil
}
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
resolvedSchemaPath, err := v.resolveSchemaPath(ctx, contract.SchemaPath)
if err != nil {
return nil, err
}
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
schemaDocument, err := loadJSONSchemaFile(ctx, resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
@@ -94,12 +93,24 @@ func (v *StandardValidator) PrepareValidation(ctx context.Context, contract doma
if err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
if err := ctx.Err(); err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{ctx: ctx, root: schemaRoot})
resourceURL := fileSchemaResourceURL(resolvedSchemaPath)
if err := ctx.Err(); err != nil {
return nil, err
}
if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
}
schema, err := compiler.Compile(resourceURL.String())
if err := ctx.Err(); err != nil {
return nil, err
}
schema, err := compileJSONSchema(ctx, resourceURL.String(), compiler.Compile)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
@@ -122,16 +133,25 @@ func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.Out
return prepared, nil
}
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
schemaName, schemaDocument, err := v.loadSchemaDocument(ctx, contract.SchemaPath)
if err != nil {
return nil, err
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
compiler := newSchemaCompiler(fsSchemaLoader{ctx: ctx, fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
if err := ctx.Err(); err != nil {
return nil, err
}
if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL.String())
if err := ctx.Err(); err != nil {
return nil, err
}
schema, err := compileJSONSchema(ctx, resourceURL.String(), compiler.Compile)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
@@ -144,7 +164,7 @@ func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.Out
return prepared, nil
}
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
type schemaValidatorFunc func(ctx context.Context, instance any, schemaPath string) ([]string, error)
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
select {
@@ -169,7 +189,11 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
res.IsValid = true
return res, nil
case domain.ValidationBasic:
if strings.TrimSpace(string(artifact.Body)) == "" {
empty := strings.TrimSpace(string(artifact.Body)) == ""
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
if empty {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"output is empty"}
@@ -179,7 +203,11 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
res.IsValid = true
return res, nil
case domain.ValidationJSON:
if !json.Valid(artifact.Body) {
valid := json.Valid(artifact.Body)
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
if !valid {
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{"invalid JSON"}
@@ -189,15 +217,18 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
res.IsValid = true
return res, nil
case domain.ValidationJSONSchema:
instance, jsonErr := decodeJSONValue(artifact.Body)
instance, jsonErr := decodeJSONValue(ctx, artifact.Body)
if jsonErr != nil {
if contextErr := ctx.Err(); contextErr != nil {
return domain.ValidationResult{}, contextErr
}
res.Status = domain.ValidationFailed
res.IsValid = false
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
return res, nil
}
validationErrors, err := validateSchema(instance, contract.SchemaPath)
validationErrors, err := validateSchema(ctx, instance, contract.SchemaPath)
if err != nil {
return domain.ValidationResult{}, err
}
@@ -216,8 +247,8 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
}
}
func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath)
func (v *StandardValidator) validateJSONSchema(ctx context.Context, instance any, schemaPath string) ([]string, error) {
resolvedSchemaPath, err := v.resolveSchemaPath(ctx, schemaPath)
if err != nil {
return nil, err
}
@@ -226,21 +257,21 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string)
if err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
if err := ctx.Err(); err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{ctx: ctx, root: schemaRoot})
resourceURL := fileSchemaResourceURL(resolvedSchemaPath)
schema, err := compiler.Compile(resourceURL.String())
schema, err := compileJSONSchema(ctx, resourceURL.String(), compiler.Compile)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
if err := schema.Validate(instance); err != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
}
return nil, nil
return executeJSONSchema(ctx, schema, instance)
}
func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath)
func (v *FSValidator) validateJSONSchema(ctx context.Context, instance any, schemaPath string) ([]string, error) {
schemaName, schemaDoc, err := v.loadSchemaDocument(ctx, schemaPath)
if err != nil {
return nil, err
}
@@ -249,40 +280,60 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str
if err := validateSchemaDialect(schemaDoc); err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
compiler := newSchemaCompiler(fsSchemaLoader{ctx: ctx, fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
if err := ctx.Err(); err != nil {
return nil, err
}
if err := compiler.AddResource(resourceURL.String(), schemaDoc); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL.String())
if err := ctx.Err(); err != nil {
return nil, err
}
schema, err := compileJSONSchema(ctx, resourceURL.String(), compiler.Compile)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
if err := schema.Validate(instance); err != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
}
return nil, nil
return executeJSONSchema(ctx, schema, instance)
}
func decodeJSONValue(body []byte) (any, error) {
func decodeJSONValue(ctx context.Context, body []byte) (any, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.UseNumber()
var value any
if err := decoder.Decode(&value); err != nil {
decodeErr := decoder.Decode(&value)
if err := ctx.Err(); err != nil {
return nil, err
}
if decodeErr != nil {
return nil, decodeErr
}
var trailing any
if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) {
return value, nil
} else if err != nil {
trailingErr := decoder.Decode(&trailing)
if err := ctx.Err(); err != nil {
return nil, err
}
if errors.Is(trailingErr, io.EOF) {
return value, nil
} else if trailingErr != nil {
return nil, trailingErr
}
return nil, errors.New("multiple JSON values")
}
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
func (v *StandardValidator) resolveSchemaPath(ctx context.Context, schemaPath string) (string, error) {
if err := ctx.Err(); err != nil {
return "", err
}
if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation")
}
@@ -291,13 +342,25 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
if err != nil {
return "", err
}
if err := ctx.Err(); err != nil {
return "", err
}
resolved, err := containedFilesystemPath(root, schemaPath)
if err != nil {
return "", err
}
if err := ctx.Err(); err != nil {
return "", err
}
if _, err := os.Stat(resolved); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return "", contextErr
}
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
}
if err := ctx.Err(); err != nil {
return "", err
}
return resolved, nil
}
@@ -318,28 +381,39 @@ func (v *StandardValidator) schemaRoot() (string, error) {
return resolved, nil
}
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
resolved, err := v.resolveSchemaPath(schemaPath)
func (v *FSValidator) loadSchemaDocument(ctx context.Context, schemaPath string) (string, any, error) {
resolved, err := v.resolveSchemaPath(ctx, schemaPath)
if err != nil {
return "", nil, err
}
raw, err := fs.ReadFile(v.fsys, resolved)
raw, err := readSchemaFile(ctx, func() (fs.File, error) {
return v.fsys.Open(resolved)
})
if err != nil {
return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
}
doc, err := decodeJSONValue(raw)
doc, err := decodeJSONValue(ctx, raw)
if err != nil {
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
if err := validateSchemaDialect(doc); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return "", nil, contextErr
}
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
}
if err := ctx.Err(); err != nil {
return "", nil, err
}
return resolved, doc, nil
}
func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
func (v *FSValidator) resolveSchemaPath(ctx context.Context, schemaPath string) (string, error) {
if err := ctx.Err(); err != nil {
return "", err
}
if strings.TrimSpace(schemaPath) == "" {
return "", errors.New("schema path is required for json_schema validation")
}
@@ -350,8 +424,14 @@ func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
cleanRoot := filecatalog.CleanFSRoot(v.root)
rootInfo, err := fs.Stat(v.fsys, cleanRoot)
if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return "", contextErr
}
return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
}
if err := ctx.Err(); err != nil {
return "", err
}
var resolved string
if rootInfo.IsDir() {
@@ -372,8 +452,14 @@ func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
}
if _, err := fs.Stat(v.fsys, resolved); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return "", contextErr
}
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
}
if err := ctx.Err(); err != nil {
return "", err
}
return resolved, nil
}
@@ -411,6 +497,35 @@ func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
return compiler
}
type schemaExecutor interface {
Validate(instance any) error
}
func compileJSONSchema(ctx context.Context, resourceURL string, compile func(string) (*jsonschema.Schema, error)) (*jsonschema.Schema, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
schema, compileErr := compile(resourceURL)
if err := ctx.Err(); err != nil {
return nil, err
}
return schema, compileErr
}
func executeJSONSchema(ctx context.Context, schema schemaExecutor, instance any) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
validationErr := schema.Validate(instance)
if err := ctx.Err(); err != nil {
return nil, err
}
if validationErr != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", validationErr)}, nil
}
return nil, nil
}
func validateSchemaDialect(doc any) error {
object, ok := doc.(map[string]any)
if !ok {
@@ -431,10 +546,14 @@ func validateSchemaDialect(doc any) error {
}
type standardSchemaLoader struct {
ctx context.Context
root string
}
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
if err := l.ctx.Err(); err != nil {
return nil, err
}
parsed, err := url.Parse(resourceURL)
if err != nil || parsed.Scheme != "file" || parsed.Host != "" || parsed.RawQuery != "" || parsed.Opaque != "" {
return nil, fmt.Errorf("schema reference %q is not a contained file reference", resourceURL)
@@ -445,9 +564,15 @@ func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
}
resolved, err := containedFilesystemPath(l.root, fileName)
if err != nil {
if contextErr := l.ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, err
}
return loadJSONSchemaFile(resolved)
if err := l.ctx.Err(); err != nil {
return nil, err
}
return loadJSONSchemaFile(l.ctx, resolved)
}
func containedFilesystemPath(root, name string) (string, error) {
@@ -473,27 +598,86 @@ func containedFilesystemPath(root, name string) (string, error) {
return candidate, nil
}
func loadJSONSchemaFile(name string) (any, error) {
raw, err := os.ReadFile(name)
func loadJSONSchemaFile(ctx context.Context, name string) (any, error) {
raw, err := readSchemaFile(ctx, func() (fs.File, error) {
return os.Open(name)
})
if err != nil {
return nil, err
}
doc, err := decodeJSONValue(raw)
doc, err := decodeJSONValue(ctx, raw)
if err != nil {
return nil, err
}
if err := validateSchemaDialect(doc); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
return doc, nil
}
func readSchemaFile(ctx context.Context, open func() (fs.File, error)) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
file, openErr := open()
if err := ctx.Err(); err != nil {
if file != nil {
_ = file.Close()
}
return nil, err
}
if openErr != nil {
if file != nil {
_ = file.Close()
}
return nil, openErr
}
if file == nil {
return nil, errors.New("schema source returned a nil file")
}
defer file.Close()
var contents []byte
chunk := make([]byte, schemaReadChunkSize)
for {
if err := ctx.Err(); err != nil {
return nil, err
}
n, readErr := file.Read(chunk)
if n > 0 {
contents = append(contents, chunk[:n]...)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if errors.Is(readErr, io.EOF) {
return contents, nil
}
if readErr != nil {
return nil, readErr
}
if n == 0 {
return nil, io.ErrNoProgress
}
}
}
type fsSchemaLoader struct {
ctx context.Context
fsys fs.FS
root string
}
func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
if err := l.ctx.Err(); err != nil {
return nil, err
}
parsed, err := url.Parse(resourceURL)
if err != nil {
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
@@ -513,21 +697,35 @@ func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
rootInfo, err := fs.Stat(l.fsys, l.root)
if err != nil {
if contextErr := l.ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, err
}
if err := l.ctx.Err(); err != nil {
return nil, err
}
if !rootInfo.IsDir() && name != l.root {
return nil, fmt.Errorf("schema reference %q is outside the configured schema file", resourceURL)
}
raw, err := fs.ReadFile(l.fsys, name)
raw, err := readSchemaFile(l.ctx, func() (fs.File, error) {
return l.fsys.Open(name)
})
if err != nil {
return nil, err
}
doc, err := decodeJSONValue(raw)
doc, err := decodeJSONValue(l.ctx, raw)
if err != nil {
return nil, err
}
if err := validateSchemaDialect(doc); err != nil {
if contextErr := l.ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, err
}
if err := l.ctx.Err(); err != nil {
return nil, err
}
return doc, nil