Escape schema resources and reuse compiled plans
This commit is contained in:
@@ -25,9 +25,9 @@ contributor workflow and validation.
|
|||||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -20,8 +20,9 @@ and override semantics consumed by the runner.
|
|||||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||||
validation. The root engine supplies one immutable registry containing the
|
validation. The root engine supplies one immutable registry containing the
|
||||||
built-in backend and validated consumer additions, one engine-local run
|
built-in backend and validated consumer additions, one engine-local run
|
||||||
admitter, and a model client wrapped by the same capacity manager. Schema
|
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||||
documents are loaded through the validator's optional schema-loader interface.
|
plans and provider-facing schema metadata come from the validator's preparation
|
||||||
|
interface.
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
An output repairer can be injected internally, but the ordinary runner
|
||||||
constructor does not enable one.
|
constructor does not enable one.
|
||||||
|
|
||||||
@@ -77,7 +78,8 @@ performs only the work needed to validate routing and admission:
|
|||||||
The completion phase consumes that state without reloading the prompt,
|
The completion phase consumes that state without reloading the prompt,
|
||||||
profile, or backend:
|
profile, or backend:
|
||||||
|
|
||||||
1. load structured-output schema metadata when required;
|
1. create one operation-local validation plan and derive structured-output
|
||||||
|
schema metadata from it when required;
|
||||||
2. load and hash input artifacts;
|
2. load and hash input artifacts;
|
||||||
3. render messages and the prompt-defined session;
|
3. render messages and the prompt-defined session;
|
||||||
4. apply any direct session ID;
|
4. apply any direct session ID;
|
||||||
@@ -88,7 +90,10 @@ profile, or backend:
|
|||||||
`Run` performs backend admission between the phases. This structure preserves
|
`Run` performs backend admission between the phases. This structure preserves
|
||||||
one execution-precedence and error-ordering implementation while allowing a
|
one execution-precedence and error-ordering implementation while allowing a
|
||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations.
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
|
and discards it when the operation ends. Prepared execution stores the same
|
||||||
|
kind of plan only in its private payload.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
@@ -119,8 +124,9 @@ its `RunAdmitter` to reserve capacity for the effective backend ID. A nil
|
|||||||
admitter is an internal unlimited fallback. After successful admission, `Run`
|
admitter is an internal unlimited fallback. After successful admission, `Run`
|
||||||
immediately defers the returned release function, performs the completion
|
immediately defers the returned release function, performs the completion
|
||||||
phase, makes one initial generation call, builds the named output artifact,
|
phase, makes one initial generation call, builds the named output artifact,
|
||||||
and validates that artifact. Invalid generated content remains a validation
|
and validates that artifact with the plan compiled during completion. Invalid
|
||||||
result; an inability to perform validation is an operational error.
|
generated content remains a validation result; an inability to perform
|
||||||
|
validation is an operational error.
|
||||||
|
|
||||||
The admission lease covers completion-phase preparation, initial generation,
|
The admission lease covers completion-phase preparation, initial generation,
|
||||||
validation, every repair, and every exit. It bounds accepted work without
|
validation, every repair, and every exit. It bounds accepted work without
|
||||||
@@ -134,8 +140,8 @@ target and session ID, validation errors, prior output, and structured-output
|
|||||||
specification. The default repairer uses the same wrapped client as initial
|
specification. The default repairer uses the same wrapped client as initial
|
||||||
generation, so each repair reacquires the selected backend's active permit
|
generation, so each repair reacquires the selected backend's active permit
|
||||||
while remaining inside its original admission lease. Repair never performs a
|
while remaining inside its original admission lease. Repair never performs a
|
||||||
second bounded admission. This capability remains internal and is not a public
|
second bounded admission, and repaired outputs use the operation's existing
|
||||||
option.
|
validation plan. This capability remains internal and is not a public option.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||||
|
|||||||
@@ -121,19 +121,21 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
|||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error.
|
||||||
|
|
||||||
For executable preparation, the built-in validators create a frozen validation
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
plan. None, basic, and JSON modes retain the effective output contract without
|
basic, and JSON modes retain the effective output contract without source
|
||||||
source access. JSON Schema mode loads the root document, resolves and compiles
|
access. JSON Schema mode loads the root document once, resolves and compiles
|
||||||
every transitive reference during preparation, and retains the compiled
|
each transitive reference, and retains the compiled validator. Schema compiler
|
||||||
validator. The provider-facing structured-output metadata uses that same
|
resources use canonical escaped file or private-scheme URLs; loaders decode
|
||||||
captured root document.
|
their paths once and enforce the configured source boundary. The
|
||||||
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
|
the same plan.
|
||||||
|
|
||||||
`PrepareExecution` also completes prompt and profile selection, artifact
|
`Prepare` discards its validation plan after returning metadata. `Run` retains
|
||||||
loading and hashing, session and message rendering, and target resolution.
|
its plan for initial and repaired-output validation, then discards it with the
|
||||||
`RunPrepared` uses the retained source-derived state and validation plan; it
|
operation. `PrepareExecution` retains the plan in its private frozen payload;
|
||||||
does not reopen prompt, profile, input, or schema sources and does not rerender
|
`RunPrepared` uses that plan without reopening prompt, profile, input, or
|
||||||
the request. By contrast, ordinary `Prepare` produces a preparation value only:
|
schema sources or rerendering the request. A later ordinary `Run` always
|
||||||
a later `Run` performs its own source resolution and preparation.
|
performs fresh source resolution and preparation.
|
||||||
|
|
||||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||||
|
|||||||
230
engine_test.go
230
engine_test.go
@@ -6,14 +6,17 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/fs"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
@@ -2424,6 +2427,162 @@ func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPublicSchemaGraphErrorsHaveSourceParity(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid keyword",
|
||||||
|
files: map[string]string{"root.json": `{"type":42}`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed direct reference",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{"$ref":"child.json"}`,
|
||||||
|
"child.json": `{`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing direct reference",
|
||||||
|
files: map[string]string{"root.json": `{"$ref":"missing.json"}`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed second-level reference",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{"$ref":"child.json"}`,
|
||||||
|
"child.json": `{"$ref":"grandchild.json"}`,
|
||||||
|
"grandchild.json": `{`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing second-level reference",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{"$ref":"child.json"}`,
|
||||||
|
"child.json": `{"$ref":"missing.json"}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unsupported referenced dialect",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{"$ref":"child.json"}`,
|
||||||
|
"child.json": `{"$schema":"http://json-schema.org/draft-07/schema#","type":"integer"}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "escaping reference",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{"$ref":"../outside.json"}`,
|
||||||
|
"../outside.json": `{"type":"integer"}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "remote reference",
|
||||||
|
files: map[string]string{"root.json": `{"$ref":"https://example.test/schema.json"}`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid multi-document graph",
|
||||||
|
files: map[string]string{
|
||||||
|
"root.json": `{
|
||||||
|
"type":"object",
|
||||||
|
"required":["value"],
|
||||||
|
"properties":{"value":{"$ref":"child.json"}}
|
||||||
|
}`,
|
||||||
|
"child.json": `{"$ref":"nested/value.json"}`,
|
||||||
|
"nested/value.json": `{"type":"integer","minimum":2}`,
|
||||||
|
},
|
||||||
|
valid: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range []string{"operating system files", "fs.FS"} {
|
||||||
|
t.Run(source, func(t *testing.T) {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}}
|
||||||
|
engine := newPublicSchemaGraphEngine(t, source, tc.files, client)
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "schema.graph.prompt",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if tc.valid {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run valid schema graph: %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("validation = %+v, want passed", result.Validation)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("generation calls = %d, want 1", len(client.requests))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !errors.Is(err, promptkit.ErrValidation) {
|
||||||
|
t.Fatalf("error = %v, want ErrValidation", err)
|
||||||
|
}
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("result = %+v, want nil", result)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatalf("invalid schema reached generation: %+v", client.requests)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunReadsSchemaGraphOncePerOperation(t *testing.T) {
|
||||||
|
source := &countingSchemaFS{
|
||||||
|
FS: fstest.MapFS{
|
||||||
|
"schemas/root.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"type":"object",
|
||||||
|
"required":["value"],
|
||||||
|
"properties":{"value":{"$ref":"child.json"}}
|
||||||
|
}`)},
|
||||||
|
"schemas/child.json": &fstest.MapFile{Data: []byte(`{"$ref":"nested/value.json"}`)},
|
||||||
|
"schemas/nested/value.json": &fstest.MapFile{Data: []byte(`{"type":"integer","minimum":2}`)},
|
||||||
|
},
|
||||||
|
reads: make(map[string]int),
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(publicStructuredPromptFS("schema.count.prompt", "root.json"), "prompts"),
|
||||||
|
promptkit.WithSchemaFS(source, "schemas"),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "schema.count.prompt",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for operation := 1; operation <= 2; operation++ {
|
||||||
|
result, err := engine.Run(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run operation %d: %v", operation, err)
|
||||||
|
}
|
||||||
|
if !result.Validation.IsValid {
|
||||||
|
t.Fatalf("run operation %d validation = %+v", operation, result.Validation)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"schemas/root.json", "schemas/child.json", "schemas/nested/value.json"} {
|
||||||
|
if got := source.readCount(name); got != operation {
|
||||||
|
t.Fatalf("after operation %d, reads for %q = %d, want %d", operation, name, got, operation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) {
|
func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) {
|
||||||
const schema = `{
|
const schema = `{
|
||||||
"type": "number",
|
"type": "number",
|
||||||
@@ -2987,7 +3146,7 @@ messages:
|
|||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: ` + schemaPath + `
|
schema_path: ` + strconv.Quote(schemaPath) + `
|
||||||
repair_attempts: 0
|
repair_attempts: 0
|
||||||
`)},
|
`)},
|
||||||
}
|
}
|
||||||
@@ -3095,6 +3254,75 @@ func (r *recordingArtifactReader) Read(ctx context.Context, ref promptkit.Artifa
|
|||||||
return r.artifact, nil
|
return r.artifact, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type countingSchemaFS struct {
|
||||||
|
fs.FS
|
||||||
|
mu sync.Mutex
|
||||||
|
reads map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *countingSchemaFS) ReadFile(name string) ([]byte, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.reads[name]++
|
||||||
|
f.mu.Unlock()
|
||||||
|
return fs.ReadFile(f.FS, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *countingSchemaFS) readCount(name string) int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.reads[name]
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPublicSchemaGraphEngine(
|
||||||
|
t *testing.T,
|
||||||
|
sourceName string,
|
||||||
|
files map[string]string,
|
||||||
|
client promptkit.LLMClient,
|
||||||
|
) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
options := []promptkit.Option{
|
||||||
|
promptkit.WithPromptFS(publicStructuredPromptFS("schema.graph.prompt", "root.json"), "prompts"),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
}
|
||||||
|
config := promptkit.Config{}
|
||||||
|
|
||||||
|
switch sourceName {
|
||||||
|
case "operating system files":
|
||||||
|
workspace := t.TempDir()
|
||||||
|
schemaRoot := filepath.Join(workspace, "schemas")
|
||||||
|
if err := os.Mkdir(schemaRoot, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, body := range files {
|
||||||
|
fileName := filepath.Clean(filepath.Join(schemaRoot, filepath.FromSlash(name)))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(fileName), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fileName, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
config.SchemaDir = schemaRoot
|
||||||
|
case "fs.FS":
|
||||||
|
schemaFS := make(fstest.MapFS, len(files))
|
||||||
|
for name, body := range files {
|
||||||
|
schemaFS[path.Clean(path.Join("schemas", name))] = &fstest.MapFile{Data: []byte(body)}
|
||||||
|
}
|
||||||
|
options = append(options, promptkit.WithSchemaFS(schemaFS, "schemas"))
|
||||||
|
default:
|
||||||
|
t.Fatalf("unknown schema source %q", sourceName)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(config, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
|||||||
@@ -42,25 +42,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
operation, err := r.completePreparation(ctx, req, state)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
structuredOutput, err := r.structuredOutputFromValidationPlan(
|
executionSnapshot, err := clonePreparedRun(operation.run)
|
||||||
state.definition,
|
|
||||||
state.effectiveContract,
|
|
||||||
validationPlan,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
executionSnapshot, err := clonePreparedRun(prepared)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -76,52 +63,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
details: details,
|
details: details,
|
||||||
payload: &preparedExecutionPayload{
|
payload: &preparedExecutionPayload{
|
||||||
prepared: executionSnapshot,
|
prepared: executionSnapshot,
|
||||||
validation: validationPlan,
|
validation: operation.validation,
|
||||||
directKey: state.effectiveModel.APIKey,
|
directKey: state.effectiveModel.APIKey,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) prepareValidation(
|
|
||||||
ctx context.Context,
|
|
||||||
contract domain.OutputContract,
|
|
||||||
) (validate.PreparedValidation, error) {
|
|
||||||
if r.validator == nil {
|
|
||||||
return noOpPreparedValidation{contract: contract}, nil
|
|
||||||
}
|
|
||||||
preparer, ok := r.validator.(validate.ValidationPreparer)
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
|
||||||
}
|
|
||||||
plan, err := preparer.PrepareValidation(ctx, contract)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
|
||||||
}
|
|
||||||
if plan == nil {
|
|
||||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
|
||||||
}
|
|
||||||
return plan, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) structuredOutputFromValidationPlan(
|
|
||||||
def *domain.PromptDefinition,
|
|
||||||
contract domain.OutputContract,
|
|
||||||
plan validate.PreparedValidation,
|
|
||||||
) (*domain.StructuredOutputSpec, error) {
|
|
||||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
schemaDocument := plan.SchemaDocument()
|
|
||||||
if schemaDocument == nil {
|
|
||||||
if r.validator == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
|
||||||
}
|
|
||||||
return structuredOutputSpec(def, schemaDocument), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Details returns a fresh credential-redacted copy of the prepared run.
|
// Details returns a fresh credential-redacted copy of the prepared run.
|
||||||
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
|
|||||||
@@ -52,6 +52,12 @@ type recordingValidationPreparer struct {
|
|||||||
directValidateCalls int
|
directValidateCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type validationOnly struct{}
|
||||||
|
|
||||||
|
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
||||||
|
return domain.ValidationResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) Validate(
|
func (v *recordingValidationPreparer) Validate(
|
||||||
context.Context,
|
context.Context,
|
||||||
*domain.Artifact,
|
*domain.Artifact,
|
||||||
@@ -532,7 +538,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
|||||||
reader,
|
reader,
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
&fakeValidator{},
|
validationOnly{},
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,11 @@ type preparationState struct {
|
|||||||
start time.Time
|
start time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type preparedOperation struct {
|
||||||
|
run *domain.PreparedRun
|
||||||
|
validation validate.PreparedValidation
|
||||||
|
}
|
||||||
|
|
||||||
func NewRunner(
|
func NewRunner(
|
||||||
promptDefs promptdef.Repository,
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
@@ -137,18 +142,20 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
|
|
||||||
prepared, err := r.completePreparation(ctx, req, state)
|
operation, err := r.completePreparation(ctx, req, state)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
directAPIKey := state.effectiveModel.APIKey
|
directAPIKey := state.effectiveModel.APIKey
|
||||||
|
|
||||||
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
|
return r.executePreparedRun(ctx, operation.run, directAPIKey, runID, start, func(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
artifact *domain.Artifact,
|
artifact *domain.Artifact,
|
||||||
attemptsUsed int,
|
attemptsUsed int,
|
||||||
) (domain.ValidationResult, error) {
|
) (domain.ValidationResult, error) {
|
||||||
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
|
result, err := operation.validation.Validate(ctx, artifact)
|
||||||
|
result.RepairAttempts = attemptsUsed
|
||||||
|
return result, err
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -250,7 +257,11 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return r.completePreparation(ctx, req, state)
|
operation, err := r.completePreparation(ctx, req, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return operation.run, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) resolvePreparation(
|
func (r *Runner) resolvePreparation(
|
||||||
@@ -315,16 +326,64 @@ func (r *Runner) completePreparation(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.RunRequest,
|
req domain.RunRequest,
|
||||||
state *preparationState,
|
state *preparationState,
|
||||||
) (*domain.PreparedRun, error) {
|
) (*preparedOperation, error) {
|
||||||
structuredOutput, err := r.resolveStructuredOutput(
|
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||||
ctx,
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
structuredOutput, err := r.structuredOutputFromValidationPlan(
|
||||||
state.definition,
|
state.definition,
|
||||||
state.effectiveContract,
|
state.effectiveContract,
|
||||||
|
validationPlan,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &preparedOperation{run: prepared, validation: validationPlan}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) prepareValidation(
|
||||||
|
ctx context.Context,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
if r.validator == nil {
|
||||||
|
return noOpPreparedValidation{contract: contract}, nil
|
||||||
|
}
|
||||||
|
preparer, ok := r.validator.(validate.ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
plan, err := preparer.PrepareValidation(ctx, contract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
|
}
|
||||||
|
if plan == nil {
|
||||||
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) structuredOutputFromValidationPlan(
|
||||||
|
def *domain.PromptDefinition,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
plan validate.PreparedValidation,
|
||||||
|
) (*domain.StructuredOutputSpec, error) {
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDocument := plan.SchemaDocument()
|
||||||
|
if schemaDocument == nil {
|
||||||
|
if r.validator == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||||
|
}
|
||||||
|
return structuredOutputSpec(def, schemaDocument), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) completePreparationWithStructuredOutput(
|
func (r *Runner) completePreparationWithStructuredOutput(
|
||||||
@@ -398,24 +457,6 @@ func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error)
|
|||||||
return release, nil
|
return release, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
|
||||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
loader, ok := r.validator.(validate.SchemaDocumentLoader)
|
|
||||||
if !ok || loader == nil {
|
|
||||||
return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation)
|
|
||||||
}
|
|
||||||
|
|
||||||
schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return structuredOutputSpec(def, schemaDoc), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
|
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
|
||||||
return &domain.StructuredOutputSpec{
|
return &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
@@ -453,25 +494,6 @@ func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
|||||||
return name
|
return name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
|
|
||||||
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
|
|
||||||
return domain.ValidationResult{
|
|
||||||
Status: domain.ValidationSkipped,
|
|
||||||
Mode: contract.ValidationMode,
|
|
||||||
SchemaPath: contract.SchemaPath,
|
|
||||||
RepairAttempts: attemptsUsed,
|
|
||||||
IsValid: true,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := r.validator.Validate(ctx, artifact, contract)
|
|
||||||
if err != nil {
|
|
||||||
return domain.ValidationResult{}, err
|
|
||||||
}
|
|
||||||
res.RepairAttempts = attemptsUsed
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool {
|
func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool {
|
||||||
if r.repairer == nil {
|
if r.repairer == nil {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -191,16 +191,34 @@ func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact,
|
|||||||
return f.result, nil
|
return f.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
f.schemaLoads++
|
var schemaDocument any
|
||||||
f.schemaLoadPath = schemaPath
|
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||||
if f.schemaErr != nil {
|
f.schemaLoads++
|
||||||
return nil, f.schemaErr
|
f.schemaLoadPath = contract.SchemaPath
|
||||||
|
if f.schemaErr != nil {
|
||||||
|
return nil, f.schemaErr
|
||||||
|
}
|
||||||
|
schemaDocument = f.schemaDoc
|
||||||
|
if schemaDocument == nil {
|
||||||
|
schemaDocument = map[string]any{"type": "object"}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if f.schemaDoc != nil {
|
return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil
|
||||||
return f.schemaDoc, nil
|
}
|
||||||
}
|
|
||||||
return map[string]any{"type": "object"}, nil
|
type fakePreparedValidator struct {
|
||||||
|
validator *fakeValidator
|
||||||
|
contract domain.OutputContract
|
||||||
|
schemaDocument any
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||||
|
return p.validator.Validate(ctx, artifact, p.contract)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *fakePreparedValidator) SchemaDocument() any {
|
||||||
|
return p.schemaDocument
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeRepairer struct {
|
type fakeRepairer struct {
|
||||||
@@ -2278,6 +2296,9 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
|||||||
if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" {
|
if repairer.reqs[0].StructuredOutput.JSONSchema.Name != "p_1" {
|
||||||
t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name)
|
t.Fatalf("expected derived schema name p_1, got %q", repairer.reqs[0].StructuredOutput.JSONSchema.Name)
|
||||||
}
|
}
|
||||||
|
if validator.schemaLoads != 1 || validator.validateCalls != 2 {
|
||||||
|
t.Fatalf("schema preparation/validation calls = (%d, %d), want (1, 2)", validator.schemaLoads, validator.validateCalls)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
|
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
@@ -94,10 +95,11 @@ func (v *StandardValidator) PrepareValidation(ctx context.Context, contract doma
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||||
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
|
resourceURL := fileSchemaResourceURL(resolvedSchemaPath)
|
||||||
|
if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil {
|
||||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
}
|
}
|
||||||
schema, err := compiler.Compile(resolvedSchemaPath)
|
schema, err := compiler.Compile(resourceURL.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
}
|
}
|
||||||
@@ -126,10 +128,10 @@ func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.Out
|
|||||||
}
|
}
|
||||||
resourceURL := fsSchemaResourceURL(schemaName)
|
resourceURL := fsSchemaResourceURL(schemaName)
|
||||||
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||||
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
|
if err := compiler.AddResource(resourceURL.String(), schemaDocument); err != nil {
|
||||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||||
}
|
}
|
||||||
schema, err := compiler.Compile(resourceURL)
|
schema, err := compiler.Compile(resourceURL.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
}
|
}
|
||||||
@@ -225,7 +227,8 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string)
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||||
schema, err := compiler.Compile(resolvedSchemaPath)
|
resourceURL := fileSchemaResourceURL(resolvedSchemaPath)
|
||||||
|
schema, err := compiler.Compile(resourceURL.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
}
|
}
|
||||||
@@ -247,10 +250,10 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str
|
|||||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
}
|
}
|
||||||
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||||
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
|
if err := compiler.AddResource(resourceURL.String(), schemaDoc); err != nil {
|
||||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||||
}
|
}
|
||||||
schema, err := compiler.Compile(resourceURL)
|
schema, err := compiler.Compile(resourceURL.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
}
|
}
|
||||||
@@ -279,47 +282,6 @@ func decodeJSONValue(body []byte) (any, error) {
|
|||||||
return nil, errors.New("multiple JSON values")
|
return nil, errors.New("multiple JSON values")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved, err := v.resolveSchemaPath(schemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, err := os.ReadFile(resolved)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
doc, err := decodeJSONValue(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
if err := validateSchemaDialect(doc); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
return doc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *FSValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
_, doc, err := v.loadSchemaDocument(schemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return doc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
||||||
if strings.TrimSpace(schemaPath) == "" {
|
if strings.TrimSpace(schemaPath) == "" {
|
||||||
return "", errors.New("schema path is required for json_schema validation")
|
return "", errors.New("schema path is required for json_schema validation")
|
||||||
@@ -427,8 +389,19 @@ func cleanSchemaFSPath(schemaPath string) (string, error) {
|
|||||||
return cleaned, nil
|
return cleaned, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fsSchemaResourceURL(schemaName string) string {
|
func fileSchemaResourceURL(schemaName string) *url.URL {
|
||||||
return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
|
filePath := filepath.ToSlash(schemaName)
|
||||||
|
if runtime.GOOS == "windows" && !strings.HasPrefix(filePath, "/") {
|
||||||
|
filePath = "/" + filePath
|
||||||
|
}
|
||||||
|
return &url.URL{Scheme: "file", Path: filePath}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fsSchemaResourceURL(schemaName string) *url.URL {
|
||||||
|
return &url.URL{
|
||||||
|
Scheme: "promptkit-schema",
|
||||||
|
Path: "/" + strings.TrimPrefix(path.Clean(schemaName), "/"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
|
func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
|
||||||
@@ -462,6 +435,10 @@ type standardSchemaLoader struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
|
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
|
||||||
|
parsed, err := url.Parse(resourceURL)
|
||||||
|
if err != nil || parsed.Scheme != "file" || parsed.Host != "" || parsed.RawQuery != "" || parsed.Opaque != "" {
|
||||||
|
return nil, fmt.Errorf("schema reference %q is not a contained file reference", resourceURL)
|
||||||
|
}
|
||||||
fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL)
|
fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err)
|
return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err)
|
||||||
@@ -521,13 +498,10 @@ func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
||||||
}
|
}
|
||||||
if parsed.Scheme != "promptkit-schema" || parsed.Host != "" {
|
if parsed.Scheme != "promptkit-schema" || parsed.Host != "" || parsed.RawQuery != "" || parsed.Opaque != "" {
|
||||||
return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL)
|
return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL)
|
||||||
}
|
}
|
||||||
name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/"))
|
name := strings.TrimPrefix(parsed.Path, "/")
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
|
||||||
}
|
|
||||||
name = path.Clean(name)
|
name = path.Clean(name)
|
||||||
if l.root == "." {
|
if l.root == "." {
|
||||||
if strings.HasPrefix(name, "../") || name == ".." {
|
if strings.HasPrefix(name, "../") || name == ".." {
|
||||||
|
|||||||
@@ -3,9 +3,11 @@ package validate
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -330,55 +332,6 @@ func TestStandardValidatorJSONSchemaCompilationError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string"}
|
|
||||||
}
|
|
||||||
}`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
loader, ok := v.(SchemaDocumentLoader)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
|
||||||
}
|
|
||||||
|
|
||||||
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
obj, ok := doc.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected object document, got %#v", doc)
|
|
||||||
}
|
|
||||||
if obj["type"] != "object" {
|
|
||||||
t.Fatalf("expected schema type=object, got %#v", obj["type"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
loader, ok := v.(SchemaDocumentLoader)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected decode error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
||||||
v := NewFSValidator(fstest.MapFS{
|
v := NewFSValidator(fstest.MapFS{
|
||||||
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
|
"schemas/events.schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
@@ -461,17 +414,70 @@ func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
func TestFSValidatorEscapesSchemaResourcePath(t *testing.T) {
|
||||||
v := NewFSValidator(fstest.MapFS{
|
for _, schemaName := range []string{"%zz.json", "space name.json", "hash#.json", "query?.json", "rún.json"} {
|
||||||
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
t.Run(schemaName, func(t *testing.T) {
|
||||||
}, "schemas")
|
v := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/" + schemaName: &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
|
}, "schemas")
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: schemaName,
|
||||||
|
})
|
||||||
|
if err != nil || !res.IsValid {
|
||||||
|
t.Fatalf("validate schema %q: result=%#v error=%v", schemaName, res, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchemaReferencesPreserveEscapedFilenames(t *testing.T) {
|
||||||
|
for _, name := range []string{"%2F.json", "space name.json", "hash#.json", "query?.json", "rún.json"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
rootName := "root-" + name
|
||||||
|
childName := "child-" + name
|
||||||
|
childPath := "nested/" + childName
|
||||||
|
reference := (&url.URL{Path: childPath}).EscapedPath()
|
||||||
|
rootSchema := []byte(`{"$ref":` + strconv.Quote(reference) + `}`)
|
||||||
|
childSchema := []byte(`{"type":"integer","minimum":2}`)
|
||||||
|
|
||||||
|
t.Run("fs.FS", func(t *testing.T) {
|
||||||
|
validator := NewFSValidator(fstest.MapFS{
|
||||||
|
"schemas/" + rootName: &fstest.MapFile{Data: rootSchema},
|
||||||
|
"schemas/" + childPath: &fstest.MapFile{Data: childSchema},
|
||||||
|
}, "schemas")
|
||||||
|
assertSchemaValidation(t, validator, rootName)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("operating system files", func(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" && strings.ContainsAny(rootName+childName, `<>:"/\|?*`) {
|
||||||
|
t.Skip("filename is not legal on Windows")
|
||||||
|
}
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, rootName), rootSchema, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(childPath)), childSchema, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertSchemaValidation(t, NewStandardValidator(root), rootName)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSchemaValidation(t *testing.T, validator Validator, schemaPath string) {
|
||||||
|
t.Helper()
|
||||||
|
result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`2`)}, domain.OutputContract{
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
SchemaPath: "%zz.json",
|
SchemaPath: schemaPath,
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "failed to register JSON schema") {
|
if err != nil || !result.IsValid {
|
||||||
t.Fatalf("expected schema registration error, got result=%#v error=%v", res, err)
|
t.Fatalf("validate schema %q: result=%#v error=%v", schemaPath, result, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -561,25 +567,6 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFSValidatorLoadSchemaDocument(t *testing.T) {
|
|
||||||
v := NewFSValidator(fstest.MapFS{
|
|
||||||
"schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
|
||||||
}, "schemas")
|
|
||||||
loader, ok := v.(SchemaDocumentLoader)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("fs validator must implement SchemaDocumentLoader")
|
|
||||||
}
|
|
||||||
|
|
||||||
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
obj, ok := doc.(map[string]any)
|
|
||||||
if !ok || obj["type"] != "object" {
|
|
||||||
t.Fatalf("unexpected schema document: %#v", doc)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{
|
if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{
|
||||||
|
|||||||
@@ -24,8 +24,3 @@ type PreparedValidation interface {
|
|||||||
type ValidationPreparer interface {
|
type ValidationPreparer interface {
|
||||||
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
|
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
|
||||||
type SchemaDocumentLoader interface {
|
|
||||||
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user