package validate import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "io/fs" "net/url" "os" "path" "path/filepath" "strings" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" "github.com/santhosh-tekuri/jsonschema/v6" ) const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema" // StandardValidator provides basic, JSON, and JSON Schema output validation. type StandardValidator struct { schemaBaseDir string } type FSValidator struct { fsys fs.FS root string } func NewStandardValidator(schemaBaseDir string) Validator { return &StandardValidator{schemaBaseDir: schemaBaseDir} } func NewFSValidator(fsys fs.FS, root string) Validator { return &FSValidator{fsys: fsys, root: root} } func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) } func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) } type preparedValidation struct { contract domain.OutputContract schemaDocument any schema *jsonschema.Schema } func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) { return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema) } func (p *preparedValidation) SchemaDocument() any { return p.schemaDocument } func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) { if p.schema == nil { return nil, errors.New("prepared JSON schema is unavailable") } if err := p.schema.Validate(instance); err != nil { return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil } return nil, nil } func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) { if err := ctx.Err(); err != nil { return nil, err } prepared := &preparedValidation{contract: contract} if contract.ValidationMode != domain.ValidationJSONSchema { return prepared, nil } resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath) if err != nil { return nil, err } schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) } schemaRoot, err := v.schemaRoot() if err != nil { return nil, err } compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot}) if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err) } schema, err := compiler.Compile(resolvedSchemaPath) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) } if err := ctx.Err(); err != nil { return nil, err } prepared.schemaDocument = schemaDocument prepared.schema = schema return prepared, nil } func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) { if err := ctx.Err(); err != nil { return nil, err } prepared := &preparedValidation{contract: contract} if contract.ValidationMode != domain.ValidationJSONSchema { return prepared, nil } schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath) if err != nil { return nil, err } resourceURL := fsSchemaResourceURL(schemaName) compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)}) if err := compiler.AddResource(resourceURL, schemaDocument); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) } schema, err := compiler.Compile(resourceURL) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } if err := ctx.Err(); err != nil { return nil, err } prepared.schemaDocument = schemaDocument prepared.schema = schema return prepared, nil } type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error) func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) { select { case <-ctx.Done(): return domain.ValidationResult{}, ctx.Err() default: } res := domain.ValidationResult{ Mode: contract.ValidationMode, SchemaPath: contract.SchemaPath, RepairAttempts: contract.RepairAttempts, } if artifact == nil { return domain.ValidationResult{}, errors.New("artifact is required for validation") } switch contract.ValidationMode { case domain.ValidationNone: res.Status = domain.ValidationSkipped res.IsValid = true return res, nil case domain.ValidationBasic: if strings.TrimSpace(string(artifact.Body)) == "" { res.Status = domain.ValidationFailed res.IsValid = false res.Errors = []string{"output is empty"} return res, nil } res.Status = domain.ValidationPassed res.IsValid = true return res, nil case domain.ValidationJSON: if !json.Valid(artifact.Body) { res.Status = domain.ValidationFailed res.IsValid = false res.Errors = []string{"invalid JSON"} return res, nil } res.Status = domain.ValidationPassed res.IsValid = true return res, nil case domain.ValidationJSONSchema: instance, jsonErr := decodeJSONValue(artifact.Body) if jsonErr != nil { res.Status = domain.ValidationFailed res.IsValid = false res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} return res, nil } validationErrors, err := validateSchema(instance, contract.SchemaPath) if err != nil { return domain.ValidationResult{}, err } if len(validationErrors) > 0 { res.Status = domain.ValidationFailed res.IsValid = false res.Errors = validationErrors return res, nil } res.Status = domain.ValidationPassed res.IsValid = true return res, nil default: return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode) } } func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath) if err != nil { return nil, err } schemaRoot, err := v.schemaRoot() if err != nil { return nil, err } compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot}) schema, err := compiler.Compile(resolvedSchemaPath) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) } if err := schema.Validate(instance); err != nil { return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil } return nil, nil } func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath) if err != nil { return nil, err } resourceURL := fsSchemaResourceURL(schemaName) if err := validateSchemaDialect(schemaDoc); err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)}) if err := compiler.AddResource(resourceURL, schemaDoc); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) } schema, err := compiler.Compile(resourceURL) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } if err := schema.Validate(instance); err != nil { return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil } return nil, nil } func decodeJSONValue(body []byte) (any, error) { decoder := json.NewDecoder(bytes.NewReader(body)) decoder.UseNumber() var value any if err := decoder.Decode(&value); err != nil { return nil, err } var trailing any if err := decoder.Decode(&trailing); errors.Is(err, io.EOF) { return value, nil } else if err != nil { return nil, err } return nil, errors.New("multiple JSON values") } func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { select { case <-ctx.Done(): return nil, ctx.Err() default: } resolved, err := v.resolveSchemaPath(schemaPath) if err != nil { return nil, err } raw, err := os.ReadFile(resolved) if err != nil { return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) } doc, err := decodeJSONValue(raw) if err != nil { return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } if err := validateSchemaDialect(doc); err != nil { return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } return doc, nil } func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { select { case <-ctx.Done(): return nil, ctx.Err() default: } _, doc, err := v.loadSchemaDocument(schemaPath) if err != nil { return nil, err } return doc, nil } func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { if strings.TrimSpace(schemaPath) == "" { return "", errors.New("schema path is required for json_schema validation") } root, err := v.schemaRoot() if err != nil { return "", err } resolved, err := containedFilesystemPath(root, schemaPath) if err != nil { return "", err } if _, err := os.Stat(resolved); err != nil { return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) } return resolved, nil } func (v *StandardValidator) schemaRoot() (string, error) { root := v.schemaBaseDir if strings.TrimSpace(root) == "" { root = "." } absolute, err := filepath.Abs(root) if err != nil { return "", fmt.Errorf("failed to resolve schema source %q: %w", root, err) } resolved, err := filepath.EvalSymlinks(absolute) if err != nil { return "", fmt.Errorf("failed to access schema source %q: %w", root, err) } return resolved, nil } func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) { resolved, err := v.resolveSchemaPath(schemaPath) if err != nil { return "", nil, err } raw, err := fs.ReadFile(v.fsys, resolved) if err != nil { return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) } doc, err := decodeJSONValue(raw) if err != nil { return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } if err := validateSchemaDialect(doc); err != nil { return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) } return resolved, doc, nil } func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) { if strings.TrimSpace(schemaPath) == "" { return "", errors.New("schema path is required for json_schema validation") } if v.fsys == nil { return "", errors.New("schema filesystem is nil") } cleanRoot := filecatalog.CleanFSRoot(v.root) rootInfo, err := fs.Stat(v.fsys, cleanRoot) if err != nil { return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err) } var resolved string if rootInfo.IsDir() { resolvedPath, _, err := filecatalog.ResolveFSPath(cleanRoot, cleanRoot, schemaPath) if err != nil { return "", err } resolved = resolvedPath } else { cleanSchemaPath, err := cleanSchemaFSPath(schemaPath) if err != nil { return "", err } if cleanSchemaPath != strings.TrimSpace(path.Base(cleanRoot)) { return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot)) } resolved = cleanRoot } if _, err := fs.Stat(v.fsys, resolved); err != nil { return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) } return resolved, nil } func cleanSchemaFSPath(schemaPath string) (string, error) { cleaned := strings.TrimSpace(schemaPath) if cleaned == "" { return "", errors.New("schema path is required for json_schema validation") } cleaned = path.Clean(cleaned) if path.IsAbs(cleaned) { return "", fmt.Errorf("schema path %q must be relative", schemaPath) } return cleaned, nil } func fsSchemaResourceURL(schemaName string) string { return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/") } func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler { compiler := jsonschema.NewCompiler() compiler.DefaultDraft(jsonschema.Draft2020) compiler.UseLoader(loader) return compiler } func validateSchemaDialect(doc any) error { object, ok := doc.(map[string]any) if !ok { return nil } value, ok := object["$schema"] if !ok { return nil } dialect, ok := value.(string) if !ok { return errors.New("$schema must be a string") } if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" { return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020) } return nil } type standardSchemaLoader struct { root string } func (l standardSchemaLoader) Load(resourceURL string) (any, error) { fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL) if err != nil { return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err) } resolved, err := containedFilesystemPath(l.root, fileName) if err != nil { return nil, err } return loadJSONSchemaFile(resolved) } func containedFilesystemPath(root, name string) (string, error) { candidate := name if !filepath.IsAbs(candidate) { candidate = filepath.Join(root, candidate) } candidate, err := filepath.Abs(candidate) if err != nil { return "", fmt.Errorf("failed to resolve schema path %q: %w", name, err) } candidate, err = filepath.EvalSymlinks(candidate) if err != nil { return "", fmt.Errorf("failed to access schema file %q: %w", candidate, err) } relative, err := filepath.Rel(root, candidate) if err != nil { return "", fmt.Errorf("failed to compare schema path %q with source root: %w", candidate, err) } if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { return "", fmt.Errorf("schema path %q escapes source root", name) } return candidate, nil } func loadJSONSchemaFile(name string) (any, error) { raw, err := os.ReadFile(name) if err != nil { return nil, err } doc, err := decodeJSONValue(raw) if err != nil { return nil, err } if err := validateSchemaDialect(doc); err != nil { return nil, err } return doc, nil } type fsSchemaLoader struct { fsys fs.FS root string } func (l fsSchemaLoader) Load(resourceURL string) (any, error) { parsed, err := url.Parse(resourceURL) if err != nil { return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err) } if parsed.Scheme != "promptkit-schema" || parsed.Host != "" { return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL) } name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/")) if err != nil { return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err) } name = path.Clean(name) if l.root == "." { if strings.HasPrefix(name, "../") || name == ".." { return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL) } } else if name != l.root && !strings.HasPrefix(name, l.root+"/") { return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL) } rootInfo, err := fs.Stat(l.fsys, l.root) if err != nil { return nil, err } if !rootInfo.IsDir() && name != l.root { return nil, fmt.Errorf("schema reference %q is outside the configured schema file", resourceURL) } raw, err := fs.ReadFile(l.fsys, name) if err != nil { return nil, err } doc, err := decodeJSONValue(raw) if err != nil { return nil, err } if err := validateSchemaDialect(doc); err != nil { return nil, err } return doc, nil }