diff --git a/docs/internal/overview.md b/docs/internal/overview.md index c55d856..4c93054 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -25,9 +25,9 @@ contributor workflow and validation. | `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | -| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | +| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | -| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | +| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | The root package assembles these internal components without exposing their representations. Consumers depend only on the root facade. diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 1014cce..ae36fcd 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -20,8 +20,9 @@ and override semantics consumed by the runner. profiles, backend resolution, artifacts, rendering, model generation, and validation. The root engine supplies one immutable registry containing the built-in backend and validated consumer additions, one engine-local run -admitter, and a model client wrapped by the same capacity manager. Schema -documents are loaded through the validator's optional schema-loader interface. +admitter, and a model client wrapped by the same capacity manager. Validation +plans and provider-facing schema metadata come from the validator's preparation +interface. An output repairer can be injected internally, but the ordinary runner constructor does not enable one. @@ -77,7 +78,8 @@ performs only the work needed to validate routing and admission: The completion phase consumes that state without reloading the prompt, profile, or backend: -1. load structured-output schema metadata when required; +1. create one operation-local validation plan and derive structured-output + schema metadata from it when required; 2. load and hash input artifacts; 3. render messages and the prompt-defined session; 4. apply any direct session ID; @@ -88,7 +90,10 @@ profile, or backend: `Run` performs backend admission between the phases. This structure preserves one execution-precedence and error-ordering implementation while allowing a full backend pool to reject work before expensive schema, artifact, and -rendering operations. +rendering operations. `Prepare` discards the plan after returning its public +metadata. `Run` retains the plan through initial and repaired-output validation +and discards it when the operation ends. Prepared execution stores the same +kind of plan only in its private payload. Pointer-based numeric overrides preserve an explicit zero. Invalid negative or out-of-range values fail as invalid requests. Endpoint overrides do not change @@ -119,8 +124,9 @@ its `RunAdmitter` to reserve capacity for the effective backend ID. A nil admitter is an internal unlimited fallback. After successful admission, `Run` immediately defers the returned release function, performs the completion phase, makes one initial generation call, builds the named output artifact, -and validates that artifact. Invalid generated content remains a validation -result; an inability to perform validation is an operational error. +and validates that artifact with the plan compiled during completion. Invalid +generated content remains a validation result; an inability to perform +validation is an operational error. The admission lease covers completion-phase preparation, initial generation, validation, every repair, and every exit. It bounds accepted work without @@ -134,8 +140,8 @@ target and session ID, validation errors, prior output, and structured-output specification. The default repairer uses the same wrapped client as initial generation, so each repair reacquires the selected backend's active permit while remaining inside its original admission lease. Repair never performs a -second bounded admission. This capability remains internal and is not a public -option. +second bounded admission, and repaired outputs use the operation's existing +validation plan. This capability remains internal and is not a public option. A successful result includes the output artifact and raw output, validation state, effective session ID, prompt and rendered-prompt hashes, selected diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 3989b68..faadf3a 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -121,19 +121,21 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation result; inability to load, register, or compile a schema is an operational error. -For executable preparation, the built-in validators create a frozen validation -plan. None, basic, and JSON modes retain the effective output contract without -source access. JSON Schema mode loads the root document, resolves and compiles -every transitive reference during preparation, and retains the compiled -validator. The provider-facing structured-output metadata uses that same -captured root document. +Every preparation operation creates one operation-local validation plan. None, +basic, and JSON modes retain the effective output contract without source +access. JSON Schema mode loads the root document once, resolves and compiles +each transitive reference, and retains the compiled validator. Schema compiler +resources use canonical escaped file or private-scheme URLs; loaders decode +their paths once and enforce the configured source boundary. The +provider-facing structured-output metadata uses the root document captured by +the same plan. -`PrepareExecution` also completes prompt and profile selection, artifact -loading and hashing, session and message rendering, and target resolution. -`RunPrepared` uses the retained source-derived state and validation plan; it -does not reopen prompt, profile, input, or schema sources and does not rerender -the request. By contrast, ordinary `Prepare` produces a preparation value only: -a later `Run` performs its own source resolution and preparation. +`Prepare` discards its validation plan after returning metadata. `Run` retains +its plan for initial and repaired-output validation, then discards it with the +operation. `PrepareExecution` retains the plan in its private frozen payload; +`RunPrepared` uses that plan without reopening prompt, profile, input, or +schema sources or rerendering the request. A later ordinary `Run` always +performs fresh source resolution and preparation. The [validator tests](../../internal/validate/standard_validator_test.go) own basic, JSON, JSON Schema, source resolution, schema loading, compilation, diff --git a/engine_test.go b/engine_test.go index dc90f4c..18452ab 100644 --- a/engine_test.go +++ b/engine_test.go @@ -6,14 +6,17 @@ import ( "errors" "fmt" "io" + "io/fs" "math" "net/http" "net/http/httptest" "os" + "path" "path/filepath" "reflect" "strconv" "strings" + "sync" "testing" "testing/fstest" "time" @@ -2424,6 +2427,162 @@ func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { } } +func TestPublicSchemaGraphErrorsHaveSourceParity(t *testing.T) { + tests := []struct { + name string + files map[string]string + valid bool + }{ + { + name: "invalid keyword", + files: map[string]string{"root.json": `{"type":42}`}, + }, + { + name: "malformed direct reference", + files: map[string]string{ + "root.json": `{"$ref":"child.json"}`, + "child.json": `{`, + }, + }, + { + name: "missing direct reference", + files: map[string]string{"root.json": `{"$ref":"missing.json"}`}, + }, + { + name: "malformed second-level reference", + files: map[string]string{ + "root.json": `{"$ref":"child.json"}`, + "child.json": `{"$ref":"grandchild.json"}`, + "grandchild.json": `{`, + }, + }, + { + name: "missing second-level reference", + files: map[string]string{ + "root.json": `{"$ref":"child.json"}`, + "child.json": `{"$ref":"missing.json"}`, + }, + }, + { + name: "unsupported referenced dialect", + files: map[string]string{ + "root.json": `{"$ref":"child.json"}`, + "child.json": `{"$schema":"http://json-schema.org/draft-07/schema#","type":"integer"}`, + }, + }, + { + name: "escaping reference", + files: map[string]string{ + "root.json": `{"$ref":"../outside.json"}`, + "../outside.json": `{"type":"integer"}`, + }, + }, + { + name: "remote reference", + files: map[string]string{"root.json": `{"$ref":"https://example.test/schema.json"}`}, + }, + { + name: "valid multi-document graph", + files: map[string]string{ + "root.json": `{ + "type":"object", + "required":["value"], + "properties":{"value":{"$ref":"child.json"}} +}`, + "child.json": `{"$ref":"nested/value.json"}`, + "nested/value.json": `{"type":"integer","minimum":2}`, + }, + valid: true, + }, + } + + for _, source := range []string{"operating system files", "fs.FS"} { + t.Run(source, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}} + engine := newPublicSchemaGraphEngine(t, source, tc.files, client) + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: "schema.graph.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + }) + if tc.valid { + if err != nil { + t.Fatalf("run valid schema graph: %v", err) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("validation = %+v, want passed", result.Validation) + } + if len(client.requests) != 1 { + t.Fatalf("generation calls = %d, want 1", len(client.requests)) + } + return + } + if !errors.Is(err, promptkit.ErrValidation) { + t.Fatalf("error = %v, want ErrValidation", err) + } + if result != nil { + t.Fatalf("result = %+v, want nil", result) + } + if len(client.requests) != 0 { + t.Fatalf("invalid schema reached generation: %+v", client.requests) + } + }) + } + }) + } +} + +func TestRunReadsSchemaGraphOncePerOperation(t *testing.T) { + source := &countingSchemaFS{ + FS: fstest.MapFS{ + "schemas/root.json": &fstest.MapFile{Data: []byte(`{ + "type":"object", + "required":["value"], + "properties":{"value":{"$ref":"child.json"}} +}`)}, + "schemas/child.json": &fstest.MapFile{Data: []byte(`{"$ref":"nested/value.json"}`)}, + "schemas/nested/value.json": &fstest.MapFile{Data: []byte(`{"type":"integer","minimum":2}`)}, + }, + reads: make(map[string]int), + } + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}} + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(publicStructuredPromptFS("schema.count.prompt", "root.json"), "prompts"), + promptkit.WithSchemaFS(source, "schemas"), + promptkit.WithProfiles(promptkit.Profile{ + ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", + }), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + request := promptkit.RunRequest{ + PromptID: "schema.count.prompt", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + }, + } + + for operation := 1; operation <= 2; operation++ { + result, err := engine.Run(context.Background(), request) + if err != nil { + t.Fatalf("run operation %d: %v", operation, err) + } + if !result.Validation.IsValid { + t.Fatalf("run operation %d validation = %+v", operation, result.Validation) + } + for _, name := range []string{"schemas/root.json", "schemas/child.json", "schemas/nested/value.json"} { + if got := source.readCount(name); got != operation { + t.Fatalf("after operation %d, reads for %q = %d, want %d", operation, name, got, operation) + } + } + } +} + func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) { const schema = `{ "type": "number", @@ -2987,7 +3146,7 @@ messages: output: format: json validation_mode: json_schema - schema_path: ` + schemaPath + ` + schema_path: ` + strconv.Quote(schemaPath) + ` repair_attempts: 0 `)}, } @@ -3095,6 +3254,75 @@ func (r *recordingArtifactReader) Read(ctx context.Context, ref promptkit.Artifa return r.artifact, nil } +type countingSchemaFS struct { + fs.FS + mu sync.Mutex + reads map[string]int +} + +func (f *countingSchemaFS) ReadFile(name string) ([]byte, error) { + f.mu.Lock() + f.reads[name]++ + f.mu.Unlock() + return fs.ReadFile(f.FS, name) +} + +func (f *countingSchemaFS) readCount(name string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.reads[name] +} + +func newPublicSchemaGraphEngine( + t *testing.T, + sourceName string, + files map[string]string, + client promptkit.LLMClient, +) *promptkit.Engine { + t.Helper() + options := []promptkit.Option{ + promptkit.WithPromptFS(publicStructuredPromptFS("schema.graph.prompt", "root.json"), "prompts"), + promptkit.WithProfiles(promptkit.Profile{ + ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", + }), + promptkit.WithLLMClient(client), + } + config := promptkit.Config{} + + switch sourceName { + case "operating system files": + workspace := t.TempDir() + schemaRoot := filepath.Join(workspace, "schemas") + if err := os.Mkdir(schemaRoot, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range files { + fileName := filepath.Clean(filepath.Join(schemaRoot, filepath.FromSlash(name))) + if err := os.MkdirAll(filepath.Dir(fileName), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(fileName, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + config.SchemaDir = schemaRoot + case "fs.FS": + schemaFS := make(fstest.MapFS, len(files)) + for name, body := range files { + schemaFS[path.Clean(path.Join("schemas", name))] = &fstest.MapFile{Data: []byte(body)} + } + options = append(options, promptkit.WithSchemaFS(schemaFS, "schemas")) + default: + t.Fatalf("unknown schema source %q", sourceName) + } + + engine, err := promptkit.NewEngine(config, options...) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + return engine +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { diff --git a/internal/usecase/prepared_execution.go b/internal/usecase/prepared_execution.go index aba42f6..d22b180 100644 --- a/internal/usecase/prepared_execution.go +++ b/internal/usecase/prepared_execution.go @@ -42,25 +42,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (* return nil, err } - validationPlan, err := r.prepareValidation(ctx, state.effectiveContract) + operation, err := r.completePreparation(ctx, req, state) if err != nil { return nil, err } - structuredOutput, err := r.structuredOutputFromValidationPlan( - state.definition, - state.effectiveContract, - validationPlan, - ) - if err != nil { - return nil, err - } - prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput) - if err != nil { - return nil, err - } - - executionSnapshot, err := clonePreparedRun(prepared) + executionSnapshot, err := clonePreparedRun(operation.run) if err != nil { return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err) } @@ -76,52 +63,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (* details: details, payload: &preparedExecutionPayload{ prepared: executionSnapshot, - validation: validationPlan, + validation: operation.validation, directKey: state.effectiveModel.APIKey, }, }, nil } -func (r *Runner) prepareValidation( - ctx context.Context, - contract domain.OutputContract, -) (validate.PreparedValidation, error) { - if r.validator == nil { - return noOpPreparedValidation{contract: contract}, nil - } - preparer, ok := r.validator.(validate.ValidationPreparer) - if !ok { - return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation) - } - plan, err := preparer.PrepareValidation(ctx, contract) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrValidation, err) - } - if plan == nil { - return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation) - } - return plan, nil -} - -func (r *Runner) structuredOutputFromValidationPlan( - def *domain.PromptDefinition, - contract domain.OutputContract, - plan validate.PreparedValidation, -) (*domain.StructuredOutputSpec, error) { - if contract.ValidationMode != domain.ValidationJSONSchema { - return nil, nil - } - - schemaDocument := plan.SchemaDocument() - if schemaDocument == nil { - if r.validator == nil { - return nil, nil - } - return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation) - } - return structuredOutputSpec(def, schemaDocument), nil -} - // Details returns a fresh credential-redacted copy of the prepared run. func (p *PreparedExecution) Details() *domain.PreparedRun { if p == nil { diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index 61ebcb7..427cbee 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -52,6 +52,12 @@ type recordingValidationPreparer struct { directValidateCalls int } +type validationOnly struct{} + +func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) { + return domain.ValidationResult{}, nil +} + func (v *recordingValidationPreparer) Validate( context.Context, *domain.Artifact, @@ -532,7 +538,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) { reader, defaultRenderer(), &fakeLLM{forbid: true}, - &fakeValidator{}, + validationOnly{}, nil, ) diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 35f64d4..6243839 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -71,6 +71,11 @@ type preparationState struct { start time.Time } +type preparedOperation struct { + run *domain.PreparedRun + validation validate.PreparedValidation +} + func NewRunner( promptDefs promptdef.Repository, profiles profile.Repository, @@ -137,18 +142,20 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes } defer release() - prepared, err := r.completePreparation(ctx, req, state) + operation, err := r.completePreparation(ctx, req, state) if err != nil { return nil, err } directAPIKey := state.effectiveModel.APIKey - return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func( + return r.executePreparedRun(ctx, operation.run, directAPIKey, runID, start, func( ctx context.Context, artifact *domain.Artifact, attemptsUsed int, ) (domain.ValidationResult, error) { - return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed) + result, err := operation.validation.Validate(ctx, artifact) + result.RepairAttempts = attemptsUsed + return result, err }) } @@ -250,7 +257,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr if err != nil { return nil, err } - return r.completePreparation(ctx, req, state) + operation, err := r.completePreparation(ctx, req, state) + if err != nil { + return nil, err + } + return operation.run, nil } func (r *Runner) resolvePreparation( @@ -315,16 +326,64 @@ func (r *Runner) completePreparation( ctx context.Context, req domain.RunRequest, state *preparationState, -) (*domain.PreparedRun, error) { - structuredOutput, err := r.resolveStructuredOutput( - ctx, +) (*preparedOperation, error) { + validationPlan, err := r.prepareValidation(ctx, state.effectiveContract) + if err != nil { + return nil, err + } + structuredOutput, err := r.structuredOutputFromValidationPlan( state.definition, state.effectiveContract, + validationPlan, ) if err != nil { return nil, err } - return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput) + prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput) + if err != nil { + return nil, err + } + return &preparedOperation{run: prepared, validation: validationPlan}, nil +} + +func (r *Runner) prepareValidation( + ctx context.Context, + contract domain.OutputContract, +) (validate.PreparedValidation, error) { + if r.validator == nil { + return noOpPreparedValidation{contract: contract}, nil + } + preparer, ok := r.validator.(validate.ValidationPreparer) + if !ok { + return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation) + } + plan, err := preparer.PrepareValidation(ctx, contract) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrValidation, err) + } + if plan == nil { + return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation) + } + return plan, nil +} + +func (r *Runner) structuredOutputFromValidationPlan( + def *domain.PromptDefinition, + contract domain.OutputContract, + plan validate.PreparedValidation, +) (*domain.StructuredOutputSpec, error) { + if contract.ValidationMode != domain.ValidationJSONSchema { + return nil, nil + } + + schemaDocument := plan.SchemaDocument() + if schemaDocument == nil { + if r.validator == nil { + return nil, nil + } + return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation) + } + return structuredOutputSpec(def, schemaDocument), nil } func (r *Runner) completePreparationWithStructuredOutput( @@ -398,24 +457,6 @@ func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) return release, nil } -func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) { - if contract.ValidationMode != domain.ValidationJSONSchema { - return nil, nil - } - - loader, ok := r.validator.(validate.SchemaDocumentLoader) - if !ok || loader == nil { - return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation) - } - - schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath) - if err != nil { - return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err) - } - - return structuredOutputSpec(def, schemaDoc), nil -} - func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec { return &domain.StructuredOutputSpec{ Type: domain.StructuredOutputJSONSchema, @@ -453,25 +494,6 @@ func deriveStructuredSchemaName(promptID string, promptVersion string) string { return name } -func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) { - if r.validator == nil || contract.ValidationMode == domain.ValidationNone { - return domain.ValidationResult{ - Status: domain.ValidationSkipped, - Mode: contract.ValidationMode, - SchemaPath: contract.SchemaPath, - RepairAttempts: attemptsUsed, - IsValid: true, - }, nil - } - - res, err := r.validator.Validate(ctx, artifact, contract) - if err != nil { - return domain.ValidationResult{}, err - } - res.RepairAttempts = attemptsUsed - return res, nil -} - func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool { if r.repairer == nil { return false diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 03cb883..edcb083 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -191,16 +191,34 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, return f.result, nil } -func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - f.schemaLoads++ - f.schemaLoadPath = schemaPath - if f.schemaErr != nil { - return nil, f.schemaErr +func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) { + var schemaDocument any + if contract.ValidationMode == domain.ValidationJSONSchema { + f.schemaLoads++ + f.schemaLoadPath = contract.SchemaPath + if f.schemaErr != nil { + return nil, f.schemaErr + } + schemaDocument = f.schemaDoc + if schemaDocument == nil { + schemaDocument = map[string]any{"type": "object"} + } } - if f.schemaDoc != nil { - return f.schemaDoc, nil - } - return map[string]any{"type": "object"}, nil + return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil +} + +type fakePreparedValidator struct { + validator *fakeValidator + contract domain.OutputContract + schemaDocument any +} + +func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) { + return p.validator.Validate(ctx, artifact, p.contract) +} + +func (p *fakePreparedValidator) SchemaDocument() any { + return p.schemaDocument } type fakeRepairer struct { @@ -2278,6 +2296,9 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" { t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name) } + if validator.schemaLoads != 1 || validator.validateCalls != 2 { + t.Fatalf("schema preparation/validation calls = (%d, %d), want (1, 2)", validator.schemaLoads, validator.validateCalls) + } } func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) { diff --git a/internal/validate/standard_validator.go b/internal/validate/standard_validator.go index 5e0fc8e..b4ab396 100644 --- a/internal/validate/standard_validator.go +++ b/internal/validate/standard_validator.go @@ -12,6 +12,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "gitea.maximumdirect.net/eric/promptkit/internal/domain" @@ -94,10 +95,11 @@ func (v *StandardValidator) PrepareValidation(ctx context.Context, contract doma return nil, err } compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot}) - if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil { + resourceURL := fileSchemaResourceURL(resolvedSchemaPath) + if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err) } - schema, err := compiler.Compile(resolvedSchemaPath) + schema, err := compiler.Compile(resourceURL.String()) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) } @@ -126,10 +128,10 @@ func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.Out } resourceURL := fsSchemaResourceURL(schemaName) compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)}) - if err := compiler.AddResource(resourceURL, schemaDocument); err != nil { + if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) } - schema, err := compiler.Compile(resourceURL) + schema, err := compiler.Compile(resourceURL.String()) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } @@ -225,7 +227,8 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string) return nil, err } compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot}) - schema, err := compiler.Compile(resolvedSchemaPath) + resourceURL := fileSchemaResourceURL(resolvedSchemaPath) + schema, err := compiler.Compile(resourceURL.String()) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err) } @@ -247,10 +250,10 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)}) - if err := compiler.AddResource(resourceURL, schemaDoc); err != nil { + if err := compiler.AddResource(resourceURL.String(), schemaDoc); err != nil { return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err) } - schema, err := compiler.Compile(resourceURL) + schema, err := compiler.Compile(resourceURL.String()) if err != nil { return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err) } @@ -279,47 +282,6 @@ func decodeJSONValue(body []byte) (any, error) { return nil, errors.New("multiple JSON values") } -func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - resolved, err := v.resolveSchemaPath(schemaPath) - if err != nil { - return nil, err - } - - raw, err := os.ReadFile(resolved) - if err != nil { - return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err) - } - - doc, err := decodeJSONValue(raw) - if err != nil { - return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) - } - if err := validateSchemaDialect(doc); err != nil { - return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err) - } - return doc, nil -} - -func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - _, doc, err := v.loadSchemaDocument(schemaPath) - if err != nil { - return nil, err - } - return doc, nil -} - func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) { if strings.TrimSpace(schemaPath) == "" { return "", errors.New("schema path is required for json_schema validation") @@ -427,8 +389,19 @@ func cleanSchemaFSPath(schemaPath string) (string, error) { return cleaned, nil } -func fsSchemaResourceURL(schemaName string) string { - return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/") +func fileSchemaResourceURL(schemaName string) *url.URL { + filePath := filepath.ToSlash(schemaName) + if runtime.GOOS == "windows" && !strings.HasPrefix(filePath, "/") { + filePath = "/" + filePath + } + return &url.URL{Scheme: "file", Path: filePath} +} + +func fsSchemaResourceURL(schemaName string) *url.URL { + return &url.URL{ + Scheme: "promptkit-schema", + Path: "/" + strings.TrimPrefix(path.Clean(schemaName), "/"), + } } func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler { @@ -462,6 +435,10 @@ type standardSchemaLoader struct { } func (l standardSchemaLoader) Load(resourceURL string) (any, error) { + parsed, err := url.Parse(resourceURL) + if err != nil || parsed.Scheme != "file" || parsed.Host != "" || parsed.RawQuery != "" || parsed.Opaque != "" { + return nil, fmt.Errorf("schema reference %q is not a contained file reference", resourceURL) + } fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL) if err != nil { return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err) @@ -521,13 +498,10 @@ func (l fsSchemaLoader) Load(resourceURL string) (any, error) { if err != nil { return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err) } - if parsed.Scheme != "promptkit-schema" || parsed.Host != "" { + if parsed.Scheme != "promptkit-schema" || parsed.Host != "" || parsed.RawQuery != "" || parsed.Opaque != "" { return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL) } - name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/")) - if err != nil { - return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err) - } + name := strings.TrimPrefix(parsed.Path, "/") name = path.Clean(name) if l.root == "." { if strings.HasPrefix(name, "../") || name == ".." { diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index ad80ffc..fc1d7c2 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -3,9 +3,11 @@ package validate import ( "context" "encoding/json" + "net/url" "os" "path/filepath" "reflect" + "runtime" "strconv" "strings" "testing" @@ -330,55 +332,6 @@ func TestStandardValidatorJSONSchemaCompilationError(t *testing.T) { } } -func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ - "type": "object", - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("standard validator must implement SchemaDocumentLoader") - } - - doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - obj, ok := doc.(map[string]any) - if !ok { - t.Fatalf("expected object document, got %#v", doc) - } - if obj["type"] != "object" { - t.Fatalf("expected schema type=object, got %#v", obj["type"]) - } -} - -func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil { - t.Fatal(err) - } - - v := NewStandardValidator(tmp) - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("standard validator must implement SchemaDocumentLoader") - } - - _, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err == nil { - t.Fatal("expected decode error") - } -} - func TestFSValidatorJSONSchemaSuccess(t *testing.T) { v := NewFSValidator(fstest.MapFS{ "schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{ @@ -461,17 +414,70 @@ func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) { } } -func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - }, "schemas") +func TestFSValidatorEscapesSchemaResourcePath(t *testing.T) { + for _, schemaName := range []string{"%zz.json", "space name.json", "hash#.json", "query?.json", "rún.json"} { + t.Run(schemaName, func(t *testing.T) { + v := NewFSValidator(fstest.MapFS{ + "schemas/" + schemaName: &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, + }, "schemas") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + ValidationMode: domain.ValidationJSONSchema, + SchemaPath: schemaName, + }) + if err != nil || !res.IsValid { + t.Fatalf("validate schema %q: result=%#v error=%v", schemaName, res, err) + } + }) + } +} + +func TestSchemaReferencesPreserveEscapedFilenames(t *testing.T) { + for _, name := range []string{"%2F.json", "space name.json", "hash#.json", "query?.json", "rún.json"} { + t.Run(name, func(t *testing.T) { + rootName := "root-" + name + childName := "child-" + name + childPath := "nested/" + childName + reference := (&url.URL{Path: childPath}).EscapedPath() + rootSchema := []byte(`{"$ref":` + strconv.Quote(reference) + `}`) + childSchema := []byte(`{"type":"integer","minimum":2}`) + + t.Run("fs.FS", func(t *testing.T) { + validator := NewFSValidator(fstest.MapFS{ + "schemas/" + rootName: &fstest.MapFile{Data: rootSchema}, + "schemas/" + childPath: &fstest.MapFile{Data: childSchema}, + }, "schemas") + assertSchemaValidation(t, validator, rootName) + }) + + t.Run("operating system files", func(t *testing.T) { + if runtime.GOOS == "windows" && strings.ContainsAny(rootName+childName, `<>:"/\|?*`) { + t.Skip("filename is not legal on Windows") + } + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, rootName), rootSchema, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(childPath)), childSchema, 0o644); err != nil { + t.Fatal(err) + } + assertSchemaValidation(t, NewStandardValidator(root), rootName) + }) + }) + } +} + +func assertSchemaValidation(t *testing.T, validator Validator, schemaPath string) { + t.Helper() + result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`2`)}, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "%zz.json", + SchemaPath: schemaPath, }) - if err == nil || !strings.Contains(err.Error(), "failed to register JSON schema") { - t.Fatalf("expected schema registration error, got result=%#v error=%v", res, err) + if err != nil || !result.IsValid { + t.Fatalf("validate schema %q: result=%#v error=%v", schemaPath, result, err) } } @@ -561,25 +567,6 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) { } } -func TestFSValidatorLoadSchemaDocument(t *testing.T) { - v := NewFSValidator(fstest.MapFS{ - "schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - }, "schemas") - loader, ok := v.(SchemaDocumentLoader) - if !ok { - t.Fatal("fs validator must implement SchemaDocumentLoader") - } - - doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - obj, ok := doc.(map[string]any) - if !ok || obj["type"] != "object" { - t.Fatalf("unexpected schema document: %#v", doc) - } -} - func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) { root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{ diff --git a/internal/validate/validator.go b/internal/validate/validator.go index 85f3b27..4de4c06 100644 --- a/internal/validate/validator.go +++ b/internal/validate/validator.go @@ -24,8 +24,3 @@ type PreparedValidation interface { type ValidationPreparer interface { PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) } - -// SchemaDocumentLoader loads JSON schema documents using validator path semantics. -type SchemaDocumentLoader interface { - LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) -}