Bind schema selections throughout execution

This commit is contained in:
2026-11-20 15:36:44 +00:00
parent 2d96dfa696
commit 6fbbaa9cfa
20 changed files with 218 additions and 66 deletions

View File

@@ -13,7 +13,7 @@ import (
type deadlineValidation struct{ deadline time.Time }
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) {
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) {
v.deadline, _ = ctx.Deadline()
return v, nil
}

View File

@@ -55,19 +55,19 @@ type recordingValidationPreparer struct {
type nilPlanPreparer struct{}
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
return nil, nil
}
func (v *recordingValidationPreparer) PrepareValidation(
_ context.Context,
contract domain.OutputContract,
input validate.Input,
) (validate.PreparedValidation, error) {
v.prepareCalls++
if v.prepareErr != nil {
return nil, v.prepareErr
}
v.plan.contract = contract
v.plan.contract = input.Contract
return v.plan, nil
}

View File

@@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate(
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
context.Context,
domain.OutputContract,
validate.Input,
) (validate.PreparedValidation, error) {
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
}

View File

@@ -18,6 +18,7 @@ import (
"go.maximumdirect.net/promptkit/internal/capacity"
"go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog"
"go.maximumdirect.net/promptkit/internal/generation"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm"
@@ -73,6 +74,7 @@ type preparationState struct {
effectiveModel domain.ExecutionTarget
targetPresence domain.ExecutionTargetPresence
effectiveContract domain.OutputContract
validationInput validate.Input
start time.Time
transientRetries uint64
}
@@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation(
promptDefinitionHash := promptSelection.hash
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
}
validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil)
if err != nil {
return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err)
}
selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" {
@@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation(
effectiveModel: effectiveModel,
targetPresence: targetPresence,
effectiveContract: effectiveContract,
validationInput: validationInput,
start: start,
transientRetries: transientRetries,
}, nil
@@ -404,7 +414,7 @@ func (r *Runner) completePreparation(
req domain.RunRequest,
state *preparationState,
) (*preparedOperation, error) {
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput)
if err != nil {
return nil, err
}
@@ -425,14 +435,15 @@ func (r *Runner) completePreparation(
func (r *Runner) prepareValidation(
ctx context.Context,
contract domain.OutputContract,
promptID string,
input validate.Input,
) (validate.PreparedValidation, error) {
if r.validator == nil {
return noOpPreparedValidation{contract: contract}, nil
return noOpPreparedValidation{contract: input.Contract}, nil
}
plan, err := r.validator.PrepareValidation(ctx, contract)
plan, err := r.validator.PrepareValidation(ctx, input)
if err != nil {
return nil, &OperationError{Category: ErrValidation, Cause: err}
return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)}
}
if plan == nil {
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
@@ -440,6 +451,31 @@ func (r *Runner) prepareValidation(
return plan, nil
}
func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) {
input := validate.Input{Contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return input, nil
}
if contract.SchemaFile != nil && !replaced {
if def.SchemaFile == nil {
return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile)
}
input.File = def.SchemaFile
return input, nil
}
if contract.SchemaPath != "" {
if def.SchemaRoot == nil {
return validate.Input{}, errors.New("schema_path requires an applicable schema_root")
}
resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath)
if err != nil {
return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err)
}
input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved}
}
return input, nil
}
func (r *Runner) structuredOutputFromValidationPlan(
def *domain.PromptDefinition,
contract domain.OutputContract,

View File

@@ -13,6 +13,7 @@ import (
"strings"
"sync"
"testing"
"testing/fstest"
"text/template"
"time"
@@ -246,7 +247,8 @@ type fakeValidator struct {
validateCalls int
}
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) {
contract := input.Contract
var schemaDocument any
if contract.ValidationMode == domain.ValidationJSONSchema {
f.schemaLoads++
@@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
staticArtifactReader{},
staticRenderer{},
scheduledClient,
validate.NewStandardValidator("."),
validate.NewValidator(),
NewDefaultOutputRepairer(scheduledClient),
admitter,
)
@@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
DefaultProfile: "exec",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."},
Validation: domain.OutputContract{
ValidationMode: mode,
RepairAttempts: attempts,

View File

@@ -0,0 +1,51 @@
package usecase
import (
"context"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
fileName := "configured.json"
def := &domain.PromptDefinition{
ID: "collection/prompt",
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"},
SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"},
Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName},
}
configured, err := bindValidationInput(def, def.Validation, false)
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
t.Fatalf("configured binding = %+v, %v", configured, err)
}
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
contract, err := resolveOutputContract(context.Background(), def, inline)
if err != nil {
t.Fatal(err)
}
selected, err := bindValidationInput(def, contract, true)
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
t.Fatalf("inline replacement = %+v, %v", selected, err)
}
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
contract, err = resolveOutputContract(context.Background(), def, path)
if err != nil {
t.Fatal(err)
}
selected, err = bindValidationInput(def, contract, true)
if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil {
t.Fatalf("path replacement = %+v, %v", selected, err)
}
def.SchemaRoot = nil
if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") {
t.Fatalf("missing root = %v", err)
}
def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}
contract.SchemaPath = "../outside.json"
if _, err := bindValidationInput(def, contract, true); err == nil {
t.Fatal("escaping path accepted")
}
}

View File

@@ -51,18 +51,17 @@ func prepareSmokeInferenceCase(
),
}, nil
case domain.SmokeTestStructuredOutput:
validator := validate.NewFSValidator(smokeSchemaFS, ".")
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
validator := validate.NewValidator()
jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
})
}})
if err != nil {
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
}
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: smokeStructuredSchemaPath,
})
}, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}})
if err != nil {
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
}

View File

@@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
type selectionTestValidator struct{ state *selectionTestCollaborators }
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
v.state.validationCalls++
return nil, errors.New("validation must not occur during target selection")
}
@@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any {
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
"session_id": "{{.vars.session}}",
"output": map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
"format": "json", "validation_mode": "json_schema", "schema": false,
},
},
map[string]any{"id": "backend.prompt",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
"format": "json", "validation_mode": "json_schema", "schema": false,
},
},
},