Add artifact reading and output validation

2026-07-28 04:23:50 +00:00
parent ebc1f3e919
commit 62b26fb29e
10 changed files with 1116 additions and 7 deletions
--- a/internal/validate/standard_validator.go
+++ b/internal/validate/standard_validator.go
@@ -0,0 +1,292 @@
+package validate
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
+	"github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+// StandardValidator provides basic, JSON, and JSON Schema output validation.
+type StandardValidator struct {
+	schemaBaseDir string
+}
+
+type FSValidator struct {
+	fsys fs.FS
+	root string
+}
+
+func NewStandardValidator(schemaBaseDir string) Validator {
+	return &StandardValidator{schemaBaseDir: schemaBaseDir}
+}
+
+func NewFSValidator(fsys fs.FS, root string) Validator {
+	return &FSValidator{fsys: fsys, root: root}
+}
+
+func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
+	return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
+}
+
+func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
+	return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
+}
+
+type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
+
+func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
+	select {
+	case <-ctx.Done():
+		return domain.ValidationResult{}, ctx.Err()
+	default:
+	}
+
+	res := domain.ValidationResult{
+		Mode:           contract.ValidationMode,
+		SchemaPath:     contract.SchemaPath,
+		RepairAttempts: contract.RepairAttempts,
+	}
+
+	if artifact == nil {
+		return domain.ValidationResult{}, errors.New("artifact is required for validation")
+	}
+
+	switch contract.ValidationMode {
+	case domain.ValidationNone:
+		res.Status = domain.ValidationSkipped
+		res.IsValid = true
+		return res, nil
+	case domain.ValidationBasic:
+		if strings.TrimSpace(string(artifact.Body)) == "" {
+			res.Status = domain.ValidationFailed
+			res.IsValid = false
+			res.Errors = []string{"output is empty"}
+			return res, nil
+		}
+		res.Status = domain.ValidationPassed
+		res.IsValid = true
+		return res, nil
+	case domain.ValidationJSON:
+		_, jsonErr := parseJSON(artifact.Body)
+		if jsonErr != nil {
+			res.Status = domain.ValidationFailed
+			res.IsValid = false
+			res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
+			return res, nil
+		}
+		res.Status = domain.ValidationPassed
+		res.IsValid = true
+		return res, nil
+	case domain.ValidationJSONSchema:
+		instance, jsonErr := parseJSON(artifact.Body)
+		if jsonErr != nil {
+			res.Status = domain.ValidationFailed
+			res.IsValid = false
+			res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
+			return res, nil
+		}
+
+		validationErrors, err := validateSchema(instance, contract.SchemaPath)
+		if err != nil {
+			return domain.ValidationResult{}, err
+		}
+		if len(validationErrors) > 0 {
+			res.Status = domain.ValidationFailed
+			res.IsValid = false
+			res.Errors = validationErrors
+			return res, nil
+		}
+
+		res.Status = domain.ValidationPassed
+		res.IsValid = true
+		return res, nil
+	default:
+		return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
+	}
+}
+
+func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
+	resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath)
+	if err != nil {
+		return nil, err
+	}
+
+	compiler := jsonschema.NewCompiler()
+	schema, err := compiler.Compile(resolvedSchemaPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
+	}
+
+	if err := schema.Validate(instance); err != nil {
+		return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
+	}
+	return nil, nil
+}
+
+func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) {
+	schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath)
+	if err != nil {
+		return nil, err
+	}
+
+	resourceURL := fsSchemaResourceURL(schemaName)
+	compiler := jsonschema.NewCompiler()
+	if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
+		return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
+	}
+	schema, err := compiler.Compile(resourceURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
+	}
+
+	if err := schema.Validate(instance); err != nil {
+		return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
+	}
+	return nil, nil
+}
+
+func parseJSON(body []byte) (any, error) {
+	var v any
+	if err := json.Unmarshal(body, &v); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
+	resolved, err := v.resolveSchemaPath(schemaPath)
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := os.ReadFile(resolved)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
+	}
+
+	var doc any
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
+	}
+	return doc, nil
+}
+
+func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
+	_, doc, err := v.loadSchemaDocument(schemaPath)
+	if err != nil {
+		return nil, err
+	}
+	return doc, nil
+}
+
+func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
+	if strings.TrimSpace(schemaPath) == "" {
+		return "", errors.New("schema path is required for json_schema validation")
+	}
+
+	resolved := schemaPath
+	if !filepath.IsAbs(schemaPath) {
+		resolved = filepath.Join(v.schemaBaseDir, schemaPath)
+	}
+
+	resolved = filepath.Clean(resolved)
+	if _, err := os.Stat(resolved); err != nil {
+		return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
+	}
+
+	return resolved, nil
+}
+
+func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
+	resolved, err := v.resolveSchemaPath(schemaPath)
+	if err != nil {
+		return "", nil, err
+	}
+
+	raw, err := fs.ReadFile(v.fsys, resolved)
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
+	}
+
+	var doc any
+	if err := json.Unmarshal(raw, &doc); err != nil {
+		return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
+	}
+	return resolved, doc, nil
+}
+
+func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) {
+	if strings.TrimSpace(schemaPath) == "" {
+		return "", errors.New("schema path is required for json_schema validation")
+	}
+	if v.fsys == nil {
+		return "", errors.New("schema filesystem is nil")
+	}
+
+	cleanRoot := filecatalog.CleanFSRoot(v.root)
+	rootInfo, err := fs.Stat(v.fsys, cleanRoot)
+	if err != nil {
+		return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err)
+	}
+
+	var resolved string
+	if rootInfo.IsDir() {
+		resolvedPath, _, err := filecatalog.ResolveFSPath(cleanRoot, cleanRoot, schemaPath)
+		if err != nil {
+			return "", err
+		}
+		resolved = resolvedPath
+	} else {
+		cleanSchemaPath, err := cleanSchemaFSPath(schemaPath)
+		if err != nil {
+			return "", err
+		}
+		if cleanSchemaPath != path.Base(cleanRoot) {
+			return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot))
+		}
+		resolved = cleanRoot
+	}
+
+	if _, err := fs.Stat(v.fsys, resolved); err != nil {
+		return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
+	}
+	return resolved, nil
+}
+
+func cleanSchemaFSPath(schemaPath string) (string, error) {
+	cleaned := strings.TrimSpace(schemaPath)
+	if cleaned == "" {
+		return "", errors.New("schema path is required for json_schema validation")
+	}
+	cleaned = path.Clean(cleaned)
+	if path.IsAbs(cleaned) {
+		return "", fmt.Errorf("schema path %q must be relative", schemaPath)
+	}
+	return cleaned, nil
+}
+
+func fsSchemaResourceURL(schemaName string) string {
+	return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
+}