diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go index 1313060..f1df5c7 100644 --- a/internal/validate/standard_validator.go +++ b/internal/validate/standard_validator.go @@ -45,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c return validateArtifact(ctx, artifact, contract, v.validateJSONSchema) } +type preparedValidation struct { + contract domain.OutputContract + schemaDocument any + schema *jsonschema.Schema +} + +func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) { + return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema) +} + +func (p *preparedValidation) SchemaDocument() any { + return p.schemaDocument +} + +func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) { + if p.schema == nil { + return nil, errors.New("prepared JSON schema is unavailable") + } + if err := p.schema.Validate(instance); err != nil { + return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil + } + return nil, nil +} + +func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + prepared := &preparedValidation{contract: contract} + if contract.ValidationMode != domain.ValidationJSONSchema { + return prepared, nil + } + + resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath) + if err != nil { + return nil, err + } + schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath) + if err != nil { + return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) + } + schemaRoot, err := v.schemaRoot() + if err != nil { + return nil, err + } + compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot}) + if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil { + return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err) + } + schema, err := compiler.Compile(resolvedSchemaPath) + if err != nil { + return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + + prepared.schemaDocument = schemaDocument + prepared.schema = schema + return prepared, nil +} + +func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + prepared := &preparedValidation{contract: contract} + if contract.ValidationMode != domain.ValidationJSONSchema { + return prepared, nil + } + + schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath) + if err != nil { + return nil, err + } + resourceURL := fsSchemaResourceURL(schemaName) + compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)}) + if err := compiler.AddResource(resourceURL, schemaDocument); err != nil { + return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) + } + schema, err := compiler.Compile(resourceURL) + if err != nil { + return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + + prepared.schemaDocument = schemaDocument + prepared.schema = schema + return prepared, nil +} + type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error) func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) { diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index ad27498..128c8d8 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strconv" "strings" "testing" @@ -120,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) { } } +func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) { + tmp := t.TempDir() + rootSchema := []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "original root", + "type": "object", + "required": ["value"], + "properties": { + "value": {"$ref": "value.json"} + } +}`) + rootPath := filepath.Join(tmp, "schema.json") + referencePath := filepath.Join(tmp, "value.json") + if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(referencePath, []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "minimum": 2 +}`), 0o644); err != nil { + t.Fatal(err) + } + + validator := NewStandardValidator(tmp) + preparer, ok := validator.(ValidationPreparer) + if !ok { + t.Fatal("standard validator does not support validation preparation") + } + prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("prepare validation: %v", err) + } + assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema) + + if err := os.Remove(rootPath); err != nil { + t.Fatal(err) + } + if err := os.Remove(referencePath); err != nil { + t.Fatal(err) + } + + valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)}) + if err != nil { + t.Fatalf("validate prepared artifact: %v", err) + } + if valid.Status != domain.ValidationPassed || !valid.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors) + } + + invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)}) + if err != nil { + t.Fatalf("validate prepared artifact: %v", err) + } + if invalid.Status != domain.ValidationFailed || invalid.IsValid { + t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid) + } +} + func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) { tmp := t.TempDir() nestedDir := filepath.Join(tmp, "dnd") @@ -295,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) { } } +func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) { + rootSchema := []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "original root", + "type": "object", + "required": ["value"], + "properties": { + "value": {"$ref": "value.json"} + } +}`) + fsys := fstest.MapFS{ + "schema.json": &fstest.MapFile{Data: rootSchema}, + "value.json": &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "minimum": 2 +}`)}, + } + validator := NewFSValidator(fsys, ".") + preparer, ok := validator.(ValidationPreparer) + if !ok { + t.Fatal("filesystem validator does not support validation preparation") + } + prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: "schema.json", + }) + if err != nil { + t.Fatalf("prepare validation: %v", err) + } + assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema) + + fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +}`)} + fsys["value.json"] = &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +}`)} + + valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)}) + if err != nil { + t.Fatalf("validate prepared artifact: %v", err) + } + if valid.Status != domain.ValidationPassed || !valid.IsValid { + t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors) + } + + invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)}) + if err != nil { + t.Fatalf("validate prepared artifact: %v", err) + } + if invalid.Status != domain.ValidationFailed || invalid.IsValid { + t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid) + } +} + func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) { v := NewFSValidator(fstest.MapFS{ "schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, @@ -548,3 +669,15 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) { }) } } + +func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) { + t.Helper() + + var expected any + if err := json.Unmarshal(expectedJSON, &expected); err != nil { + t.Fatalf("decode expected schema document: %v", err) + } + if !reflect.DeepEqual(got, expected) { + t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected) + } +} diff --git a/internal/validate/validator.go b/internal/validate/validator.go index bb44042..85f3b27 100644 --- a/internal/validate/validator.go +++ b/internal/validate/validator.go @@ -2,6 +2,7 @@ package validate import ( "context" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) @@ -10,6 +11,20 @@ type Validator interface { Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) } +// PreparedValidation validates artifacts against one frozen output contract. +type PreparedValidation interface { + Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) + // SchemaDocument returns the root JSON Schema document used for provider + // structured output, or nil for non-schema modes. Returned internal + // immutable state must not be mutated. + SchemaDocument() any +} + +// ValidationPreparer freezes validation resources for one output contract. +type ValidationPreparer interface { + PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) +} + // SchemaDocumentLoader loads JSON schema documents using validator path semantics. type SchemaDocumentLoader interface { LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)