From 62b26fb29e2cea30bd56921d704bb262b0e333ce Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 28 Jul 2026 04:23:50 +0000 Subject: [PATCH] Add artifact reading and output validation --- docs/internal/overview.md | 6 +- docs/internal/sources.md | 67 +++ docs/policy/architecture.md | 14 +- go.mod | 7 +- go.sum | 6 + internal/artifact/reader.go | 122 ++++++ internal/artifact/reader_test.go | 180 ++++++++ internal/validate/standard_validator.go | 292 +++++++++++++ internal/validate/standard_validator_test.go | 413 +++++++++++++++++++ internal/validate/validator.go | 16 + 10 files changed, 1116 insertions(+), 7 deletions(-) create mode 100644 docs/internal/sources.md create mode 100644 internal/artifact/reader.go create mode 100644 internal/artifact/reader_test.go create mode 100644 internal/validate/standard_validator.go create mode 100644 internal/validate/standard_validator_test.go create mode 100644 internal/validate/validator.go diff --git a/docs/internal/overview.md b/docs/internal/overview.md index cc2df9a..924d49e 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -19,10 +19,12 @@ contributor workflow and validation. | `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Profile repositories](../../internal/profile/filesystem_repository.go) | | `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in profile repository](../../internal/profile/builtin/repository.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | +| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | +| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Internal sources and validation](sources.md) | These packages provide the internal model, source, and rendering foundation. -Artifact reading, output validation, model clients, orchestration, and a usable -public engine are not implemented in Promptkit yet. +Model clients, orchestration, and a usable public engine are not implemented in +Promptkit yet. ## Maintenance diff --git a/docs/internal/sources.md b/docs/internal/sources.md new file mode 100644 index 0000000..338334f --- /dev/null +++ b/docs/internal/sources.md @@ -0,0 +1,67 @@ +# Internal Sources And Validation + +## Purpose + +This document describes Promptkit's implemented internal source, artifact, +rendering, and output-validation behavior. The +[architecture policy](../policy/architecture.md) owns the library boundary and +dependency rules. None of these internal packages is a supported consumer API, +and the root package does not yet assemble them into a usable engine. + +## Prompt Definitions + +`internal/promptdef` loads prompt definitions from an operating-system +filesystem or an `fs.FS`. It discovers YAML deterministically, decodes fields +strictly, validates definitions, selects an ID and optional version, and +resolves file-backed message content within the selected source. + +Its package tests own prompt selection, strict decoding, definition validation, +duplicate detection, and source containment: +[prompt-definition repository tests](../../internal/promptdef/repository_test.go). + +## Profiles And Built-Ins + +`internal/profile` loads strictly decoded execution profiles from an +operating-system filesystem or an `fs.FS`. It validates required profile data, +rejects raw API keys, and supports a primary repository with fallback only +when the primary reports that a profile is absent. + +`internal/profile/builtin` embeds the maintained built-in profile catalog and +can place a caller-selected repository ahead of that catalog. Profile behavior +is owned by the +[profile repository tests](../../internal/profile/repository_test.go), while +catalog completeness, duplicate IDs, and overlay behavior are owned by the +[built-in repository tests](../../internal/profile/builtin/repository_test.go). + +## Ordinary Artifacts + +`internal/artifact` resolves inline references and unrestricted, +caller-selected file paths. It copies content into an artifact, records +metadata and a content hash, applies a content-type fallback, and honors +context cancellation. + +This ordinary reader does not implement an inbound HTTP security boundary. In +particular, it does not constrain files to an application root or impose an +HTTP request-size policy. Scriptorium's restricted HTTP reader remains an +application concern outside Promptkit. The +[artifact reader tests](../../internal/artifact/reader_test.go) own the +implemented reader behavior and failures. + +## Rendering + +`internal/prompt` renders definition messages as Go templates using named +artifacts and variables. It carries message roles, session IDs, and cache +control into the rendered prompt. The +[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior. + +## Schemas And Output Validation + +`internal/validate` provides validators backed by an operating-system +filesystem or an `fs.FS`. Validation can be skipped, require non-empty output, +require JSON, or apply a JSON Schema loaded with the source's path semantics. +Invalid generated content is returned as a validation result; inability to +load, register, or compile a schema is an operational error. + +The [validator tests](../../internal/validate/standard_validator_test.go) own +basic, JSON, JSON Schema, source resolution, schema loading, compilation, and +content-failure behavior. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 09c976d..4bf5ec3 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -29,14 +29,20 @@ The implemented internal components consist of: - `internal/profile`, which loads, validates, and overlays execution profiles from filesystem and `fs.FS` sources; - `internal/profile/builtin`, which embeds the built-in execution profile - catalog; and -- `internal/prompt`, which renders prompt messages from Go templates. + catalog; +- `internal/prompt`, which renders prompt messages from Go templates; +- `internal/artifact`, which resolves ordinary inline and unrestricted + caller-selected file references; and +- `internal/validate`, which validates basic, JSON, and JSON Schema output + using filesystem and `fs.FS` schema sources. The defaults and renderer depend on the domain model. Prompt-definition and profile repositories use the domain model, file catalog, and YAML decoder. The built-in profile repository supplies an embedded `fs.FS` to the profile -package. Artifact reading, output validation, model clients, orchestration, and -the public engine have not yet been extracted. +package. Artifact reading uses the domain model and application-neutral +defaults. Validation uses the domain model, file catalog, and JSON Schema +implementation. Model clients, orchestration, and the public engine have not +yet been extracted. Future framework extraction must follow this dependency direction: diff --git a/go.mod b/go.mod index 83032dc..817da8d 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,9 @@ module gitea.maximumdirect.net/eric/promptkit go 1.25.5 -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + gopkg.in/yaml.v3 v3.0.1 +) + +require golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index a62c313..7e38d45 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/artifact/reader.go b/internal/artifact/reader.go new file mode 100644 index 0000000..e042a0b --- /dev/null +++ b/internal/artifact/reader.go @@ -0,0 +1,122 @@ +package artifact + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + + "gitea.maximumdirect.net/eric/promptkit/internal/defaults" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +var ( + ErrUnsupportedRefType = errors.New("unsupported artifact reference type") + ErrMissingInlineBody = errors.New("missing body for inline artifact") + ErrMissingFilePath = errors.New("missing file path for file artifact") +) + +// Reader resolves artifact references into actual artifacts. +type Reader interface { + Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) +} + +// CompositeReader routes artifact resolution based on the reference type. +type CompositeReader struct { + inlineReader *inlineReader + fileReader Reader +} + +func NewCompositeReader() Reader { + return &CompositeReader{ + inlineReader: &inlineReader{}, + fileReader: &fileReader{}, + } +} + +func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + switch ref.Type { + case domain.ArtifactRefInline: + return c.inlineReader.Read(ctx, ref) + case domain.ArtifactRefFile: + return c.fileReader.Read(ctx, ref) + default: + return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type) + } +} + +type inlineReader struct{} + +func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if ref.Body == "" { + return nil, ErrMissingInlineBody + } + + body := []byte(ref.Body) + return &domain.Artifact{ + ContentType: defaults.ContentTypeTextPlain, + Body: body, + Size: int64(len(body)), + Hash: fmt.Sprintf("%x", sha256.Sum256(body)), + URI: ref.URI, + }, nil +} + +type fileReader struct{} + +func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if ref.URI == "" { + return nil, ErrMissingFilePath + } + + return readFileArtifact(ref.URI) +} + +func readFileArtifact(path string) (*domain.Artifact, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", path, err) + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", path, err) + } + + contentType := mime.TypeByExtension(filepath.Ext(path)) + if contentType == "" { + contentType = defaults.ContentTypeTextPlain + } + + return &domain.Artifact{ + Name: filepath.Base(path), + ContentType: contentType, + Body: data, + URI: path, + Size: int64(len(data)), + Hash: fmt.Sprintf("%x", sha256.Sum256(data)), + }, nil +} diff --git a/internal/artifact/reader_test.go b/internal/artifact/reader_test.go new file mode 100644 index 0000000..901e2d7 --- /dev/null +++ b/internal/artifact/reader_test.go @@ -0,0 +1,180 @@ +package artifact + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestCompositeReader_Read(t *testing.T) { + reader := NewCompositeReader() + ctx := context.Background() + + t.Run("inline artifact", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefInline, + Body: "hello world", + } + art, err := reader.Read(ctx, ref) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(art.Body) != "hello world" { + t.Errorf("expected 'hello world', got %s", string(art.Body)) + } + if art.ContentType != "text/plain" { + t.Errorf("expected text/plain content type, got %q", art.ContentType) + } + if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" { + t.Errorf("unexpected hash: %s", art.Hash) + } + if art.Size != int64(len(ref.Body)) { + t.Errorf("expected size %d, got %d", len(ref.Body), art.Size) + } + }) + + t.Run("inline artifact missing body", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefInline, + Body: "", + } + _, err := reader.Read(ctx, ref) + if !errors.Is(err, ErrMissingInlineBody) { + t.Errorf("expected ErrMissingInlineBody, got %v", err) + } + }) + + t.Run("unsupported ref type", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefType("unsupported"), + URI: "unsupported://bucket/key", + } + _, err := reader.Read(ctx, ref) + if !errors.Is(err, ErrUnsupportedRefType) { + t.Error("expected error for unsupported type") + } + }) +} + +func TestCompositeReaderCopiesInlineData(t *testing.T) { + reader := NewCompositeReader() + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefInline, + Body: "hello", + URI: "inline:greeting", + } + + first, err := reader.Read(context.Background(), ref) + if err != nil { + t.Fatalf("read first artifact: %v", err) + } + first.Body[0] = 'j' + + second, err := reader.Read(context.Background(), ref) + if err != nil { + t.Fatalf("read second artifact: %v", err) + } + if got := string(second.Body); got != ref.Body { + t.Fatalf("expected an independent body %q, got %q", ref.Body, got) + } + if second.URI != ref.URI { + t.Fatalf("expected URI %q, got %q", ref.URI, second.URI) + } +} + +func TestCompositeReaderHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := NewCompositeReader().Read(ctx, domain.ArtifactRef{ + Type: domain.ArtifactRefInline, + Body: "ignored", + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context cancellation, got %v", err) + } +} + +func TestFileReader_Read(t *testing.T) { + content := []byte("test file content") + filePath := filepath.Join(t.TempDir(), "artifact.txt") + if err := os.WriteFile(filePath, content, 0o600); err != nil { + t.Fatal(err) + } + + reader := NewCompositeReader() + ctx := context.Background() + + t.Run("file artifact loading", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefFile, + URI: filePath, + } + art, err := reader.Read(ctx, ref) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(art.Body) != string(content) { + t.Errorf("expected %s, got %s", string(content), string(art.Body)) + } + if art.Name != filepath.Base(filePath) { + t.Errorf("expected name %q, got %q", filepath.Base(filePath), art.Name) + } + if !strings.HasPrefix(art.ContentType, "text/plain") { + t.Errorf("expected text content type, got %q", art.ContentType) + } + if art.URI != filePath { + t.Errorf("expected URI %q, got %q", filePath, art.URI) + } + if art.Size != int64(len(content)) { + t.Errorf("expected size %d, got %d", len(content), art.Size) + } + if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" { + t.Errorf("unexpected hash: %s", art.Hash) + } + }) + + t.Run("missing file path", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefFile, + URI: "", + } + _, err := reader.Read(ctx, ref) + if !errors.Is(err, ErrMissingFilePath) { + t.Errorf("expected ErrMissingFilePath, got %v", err) + } + }) + + t.Run("missing file", func(t *testing.T) { + ref := domain.ArtifactRef{ + Type: domain.ArtifactRefFile, + URI: filepath.Join(t.TempDir(), "missing.txt"), + } + if _, err := reader.Read(ctx, ref); err == nil { + t.Fatal("expected missing file error") + } + }) + + t.Run("unknown extension uses text fallback", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "artifact.unknownextension") + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatal(err) + } + + art, err := reader.Read(ctx, domain.ArtifactRef{ + Type: domain.ArtifactRefFile, + URI: path, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if art.ContentType != "text/plain" { + t.Errorf("expected text/plain fallback, got %q", art.ContentType) + } + }) +} diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go new file mode 100644 index 0000000..81624c8 --- /dev/null +++ b/internal/validate/standard_validator.go @@ -0,0 +1,292 @@ +package validate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" + "github.com/santhosh-tekuri/jsonschema/v6" +) + +// StandardValidator provides basic, JSON, and JSON Schema output validation. +type StandardValidator struct { + schemaBaseDir string +} + +type FSValidator struct { + fsys fs.FS + root string +} + +func NewStandardValidator(schemaBaseDir string) Validator { + return &StandardValidator{schemaBaseDir: schemaBaseDir} +} + +func NewFSValidator(fsys fs.FS, root string) Validator { + return &FSValidator{fsys: fsys, root: root} +} + +func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { + return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) +} + +func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { + return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) +} + +type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error) + +func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) { + select { + case <-ctx.Done(): + return domain.ValidationResult{}, ctx.Err() + default: + } + + res := domain.ValidationResult{ + Mode: contract.ValidationMode, + SchemaPath: contract.SchemaPath, + RepairAttempts: contract.RepairAttempts, + } + + if artifact == nil { + return domain.ValidationResult{}, errors.New("artifact is required for validation") + } + + switch contract.ValidationMode { + case domain.ValidationNone: + res.Status = domain.ValidationSkipped + res.IsValid = true + return res, nil + case domain.ValidationBasic: + if strings.TrimSpace(string(artifact.Body)) == "" { + res.Status = domain.ValidationFailed + res.IsValid = false + res.Errors = []string{"output is empty"} + return res, nil + } + res.Status = domain.ValidationPassed + res.IsValid = true + return res, nil + case domain.ValidationJSON: + _, jsonErr := parseJSON(artifact.Body) + if jsonErr != nil { + res.Status = domain.ValidationFailed + res.IsValid = false + res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} + return res, nil + } + res.Status = domain.ValidationPassed + res.IsValid = true + return res, nil + case domain.ValidationJSONSchema: + instance, jsonErr := parseJSON(artifact.Body) + if jsonErr != nil { + res.Status = domain.ValidationFailed + res.IsValid = false + res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)} + return res, nil + } + + validationErrors, err := validateSchema(instance, contract.SchemaPath) + if err != nil { + return domain.ValidationResult{}, err + } + if len(validationErrors) > 0 { + res.Status = domain.ValidationFailed + res.IsValid = false + res.Errors = validationErrors + return res, nil + } + + res.Status = domain.ValidationPassed + res.IsValid = true + return res, nil + default: + return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode) + } +} + +func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { + resolvedSchemaPath, err := v.resolveSchemaPath(schemaPath) + if err != nil { + return nil, err + } + + compiler := jsonschema.NewCompiler() + schema, err := compiler.Compile(resolvedSchemaPath) + if err != nil { + return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) + } + + if err := schema.Validate(instance); err != nil { + return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil + } + return nil, nil +} + +func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]string, error) { + schemaName, schemaDoc, err := v.loadSchemaDocument(schemaPath) + if err != nil { + return nil, err + } + + resourceURL := fsSchemaResourceURL(schemaName) + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource(resourceURL, schemaDoc); err != nil { + return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) + } + schema, err := compiler.Compile(resourceURL) + if err != nil { + return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) + } + + if err := schema.Validate(instance); err != nil { + return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil + } + return nil, nil +} + +func parseJSON(body []byte) (any, error) { + var v any + if err := json.Unmarshal(body, &v); err != nil { + return nil, err + } + return v, nil +} + +func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + resolved, err := v.resolveSchemaPath(schemaPath) + if err != nil { + return nil, err + } + + raw, err := os.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) + } + + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) + } + return doc, nil +} + +func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + _, doc, err := v.loadSchemaDocument(schemaPath) + if err != nil { + return nil, err + } + return doc, nil +} + +func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { + if strings.TrimSpace(schemaPath) == "" { + return "", errors.New("schema path is required for json_schema validation") + } + + resolved := schemaPath + if !filepath.IsAbs(schemaPath) { + resolved = filepath.Join(v.schemaBaseDir, schemaPath) + } + + resolved = filepath.Clean(resolved) + if _, err := os.Stat(resolved); err != nil { + return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) + } + + return resolved, nil +} + +func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) { + resolved, err := v.resolveSchemaPath(schemaPath) + if err != nil { + return "", nil, err + } + + raw, err := fs.ReadFile(v.fsys, resolved) + if err != nil { + return "", nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) + } + + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) + } + return resolved, doc, nil +} + +func (v *FSValidator) resolveSchemaPath(schemaPath string) (string, error) { + if strings.TrimSpace(schemaPath) == "" { + return "", errors.New("schema path is required for json_schema validation") + } + if v.fsys == nil { + return "", errors.New("schema filesystem is nil") + } + + cleanRoot := filecatalog.CleanFSRoot(v.root) + rootInfo, err := fs.Stat(v.fsys, cleanRoot) + if err != nil { + return "", fmt.Errorf("failed to access schema source %q: %w", cleanRoot, err) + } + + var resolved string + if rootInfo.IsDir() { + resolvedPath, _, err := filecatalog.ResolveFSPath(cleanRoot, cleanRoot, schemaPath) + if err != nil { + return "", err + } + resolved = resolvedPath + } else { + cleanSchemaPath, err := cleanSchemaFSPath(schemaPath) + if err != nil { + return "", err + } + if cleanSchemaPath != path.Base(cleanRoot) { + return "", fmt.Errorf("schema path %q does not match schema file %q", cleanSchemaPath, path.Base(cleanRoot)) + } + resolved = cleanRoot + } + + if _, err := fs.Stat(v.fsys, resolved); err != nil { + return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err) + } + return resolved, nil +} + +func cleanSchemaFSPath(schemaPath string) (string, error) { + cleaned := strings.TrimSpace(schemaPath) + if cleaned == "" { + return "", errors.New("schema path is required for json_schema validation") + } + cleaned = path.Clean(cleaned) + if path.IsAbs(cleaned) { + return "", fmt.Errorf("schema path %q must be relative", schemaPath) + } + return cleaned, nil +} + +func fsSchemaResourceURL(schemaName string) string { + return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/") +} diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go new file mode 100644 index 0000000..18ce5ae --- /dev/null +++ b/internal/validate/standard_validator_test.go @@ -0,0 +1,413 @@ +package validate + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestStandardValidatorNoneSkipped(t *testing.T) { + v := NewStandardValidator("") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}, domain.OutputContract{ + ValidationMode: domain.ValidationNone, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationSkipped { + t.Fatalf("expected skipped, got %q", res.Status) + } + if !res.IsValid { + t.Fatal("expected valid=true for skipped") + } +} + +func TestStandardValidatorBasicSuccess(t *testing.T) { + v := NewStandardValidator("") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{ + ValidationMode: domain.ValidationBasic, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } +} + +func TestStandardValidatorBasicFailureEmpty(t *testing.T) { + v := NewStandardValidator("") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{ + ValidationMode: domain.ValidationBasic, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationFailed || res.IsValid { + t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) + } + if len(res.Errors) == 0 { + t.Fatal("expected validation errors") + } +} + +func TestStandardValidatorJSONSuccess(t *testing.T) { + v := NewStandardValidator("") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSON, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } +} + +func TestStandardValidatorJSONFailure(t *testing.T) { + v := NewStandardValidator("") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSON, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationFailed || res.IsValid { + t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) + } + if len(res.Errors) == 0 { + t.Fatal("expected parse errors") + } +} + +func TestStandardValidatorJSONSchemaSuccess(t *testing.T) { + tmp := t.TempDir() + schemaPath := filepath.Join(tmp, "schema.json") + if err := os.WriteFile(schemaPath, []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } +} + +func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) { + tmp := t.TempDir() + nestedDir := filepath.Join(tmp, "dnd") + if err := os.MkdirAll(nestedDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(nestedDir, "schema.json"), []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: filepath.Join("dnd", "schema.json"), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } +} + +func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) { + v := NewStandardValidator(t.TempDir()) + + _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: filepath.Join("dnd", "missing.json"), + }) + if err == nil { + t.Fatal("expected nested schema load error") + } +} + +func TestStandardValidatorJSONSchemaFailure(t *testing.T) { + tmp := t.TempDir() + schemaPath := filepath.Join(tmp, "schema.json") + if err := os.WriteFile(schemaPath, []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationFailed || res.IsValid { + t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid) + } + if len(res.Errors) == 0 { + t.Fatal("expected schema errors") + } +} + +func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) { + v := NewStandardValidator(t.TempDir()) + + _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "missing.json", + }) + if err == nil { + t.Fatal("expected schema load error") + } +} + +func TestStandardValidatorJSONSchemaCompilationError(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{"type":42}`), 0o644); err != nil { + t.Fatal(err) + } + v := NewStandardValidator(tmp) + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err == nil || !strings.Contains(err.Error(), "failed to compile JSON schema") { + t.Fatalf("expected schema compilation error, got result=%#v error=%v", res, err) + } +} + +func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ + "type": "object", + "properties": { + "name": {"type": "string"} + } +}`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + loader, ok := v.(SchemaDocumentLoader) + if !ok { + t.Fatal("standard validator must implement SchemaDocumentLoader") + } + + doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + obj, ok := doc.(map[string]any) + if !ok { + t.Fatalf("expected object document, got %#v", doc) + } + if obj["type"] != "object" { + t.Fatalf("expected schema type=object, got %#v", obj["type"]) + } +} + +func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) { + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil { + t.Fatal(err) + } + + v := NewStandardValidator(tmp) + loader, ok := v.(SchemaDocumentLoader) + if !ok { + t.Fatal("standard validator must implement SchemaDocumentLoader") + } + + _, err := loader.LoadSchemaDocument(context.Background(), "schema.json") + if err == nil { + t.Fatal("expected decode error") + } +} + +func TestFSValidatorJSONSchemaSuccess(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["events"], + "properties": { + "events": {"type": "array"} + } +}`)}, + }, "schemas") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "events.schema.json", + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } +} + +func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + }, "schemas") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "%zz.json", + }) + if err == nil || !strings.Contains(err.Error(), "failed to register JSON schema") { + t.Fatalf("expected schema registration error, got result=%#v error=%v", res, err) + } +} + +func TestFSValidatorJSONSchemaPathContainment(t *testing.T) { + t.Run("nested schema inside root succeeds", func(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/nested/events.schema.json": &fstest.MapFile{Data: []byte(`{ + "type": "object", + "required": ["events"], + "properties": { + "events": {"type": "array"} + } +}`)}, + }, "schemas") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "nested/events.schema.json", + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } + }) + + tests := []struct { + name string + schemaPath string + wantErr string + }{ + {name: "parent escape rejected", schemaPath: "../outside.schema.json", wantErr: "escapes source root"}, + {name: "absolute path rejected", schemaPath: "/outside.schema.json", wantErr: "must be relative"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + "outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + "schemas/outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + }, "schemas") + + _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: tc.schemaPath, + }) + if err == nil { + t.Fatal("expected schema path error") + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) + } + }) + } +} + +func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "events.schema.json": &fstest.MapFile{Data: []byte(`{ + "type": "object", + "required": ["events"], + "properties": { + "events": {"type": "array"} + } +}`)}, + }, "events.schema.json") + + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "events.schema.json", + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Status != domain.ValidationPassed || !res.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) + } + + _, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "other.schema.json", + }) + if err == nil { + t.Fatal("expected schema path mismatch error") + } +} + +func TestFSValidatorLoadSchemaDocument(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + }, "schemas") + loader, ok := v.(SchemaDocumentLoader) + if !ok { + t.Fatal("fs validator must implement SchemaDocumentLoader") + } + + doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + obj, ok := doc.(map[string]any) + if !ok || obj["type"] != "object" { + t.Fatalf("unexpected schema document: %#v", doc) + } +} diff --git a/internal/validate/validator.go b/internal/validate/validator.go new file mode 100644 index 0000000..bb44042 --- /dev/null +++ b/internal/validate/validator.go @@ -0,0 +1,16 @@ +package validate + +import ( + "context" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +// Validator validates the generated artifact based on the output contract. +type Validator interface { + Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) +} + +// SchemaDocumentLoader loads JSON schema documents using validator path semantics. +type SchemaDocumentLoader interface { + LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) +}