Bind schema selections throughout execution
This commit is contained in:
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
type deadlineValidation struct{ deadline time.Time }
|
||||
|
||||
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) {
|
||||
v.deadline, _ = ctx.Deadline()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
@@ -55,19 +55,19 @@ type recordingValidationPreparer struct {
|
||||
|
||||
type nilPlanPreparer struct{}
|
||||
|
||||
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (v *recordingValidationPreparer) PrepareValidation(
|
||||
_ context.Context,
|
||||
contract domain.OutputContract,
|
||||
input validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
v.prepareCalls++
|
||||
if v.prepareErr != nil {
|
||||
return nil, v.prepareErr
|
||||
}
|
||||
v.plan.contract = contract
|
||||
v.plan.contract = input.Contract
|
||||
return v.plan, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate(
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||
context.Context,
|
||||
domain.OutputContract,
|
||||
validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||
"go.maximumdirect.net/promptkit/internal/generation"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
@@ -73,6 +74,7 @@ type preparationState struct {
|
||||
effectiveModel domain.ExecutionTarget
|
||||
targetPresence domain.ExecutionTargetPresence
|
||||
effectiveContract domain.OutputContract
|
||||
validationInput validate.Input
|
||||
start time.Time
|
||||
transientRetries uint64
|
||||
}
|
||||
@@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation(
|
||||
promptDefinitionHash := promptSelection.hash
|
||||
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
|
||||
if err != nil {
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
return nil, contextErr
|
||||
}
|
||||
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err)
|
||||
}
|
||||
|
||||
selectedProfileID := req.ProfileID
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
@@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation(
|
||||
effectiveModel: effectiveModel,
|
||||
targetPresence: targetPresence,
|
||||
effectiveContract: effectiveContract,
|
||||
validationInput: validationInput,
|
||||
start: start,
|
||||
transientRetries: transientRetries,
|
||||
}, nil
|
||||
@@ -404,7 +414,7 @@ func (r *Runner) completePreparation(
|
||||
req domain.RunRequest,
|
||||
state *preparationState,
|
||||
) (*preparedOperation, error) {
|
||||
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||
validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -425,14 +435,15 @@ func (r *Runner) completePreparation(
|
||||
|
||||
func (r *Runner) prepareValidation(
|
||||
ctx context.Context,
|
||||
contract domain.OutputContract,
|
||||
promptID string,
|
||||
input validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
if r.validator == nil {
|
||||
return noOpPreparedValidation{contract: contract}, nil
|
||||
return noOpPreparedValidation{contract: input.Contract}, nil
|
||||
}
|
||||
plan, err := r.validator.PrepareValidation(ctx, contract)
|
||||
plan, err := r.validator.PrepareValidation(ctx, input)
|
||||
if err != nil {
|
||||
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||
return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)}
|
||||
}
|
||||
if plan == nil {
|
||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||
@@ -440,6 +451,31 @@ func (r *Runner) prepareValidation(
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) {
|
||||
input := validate.Input{Contract: contract}
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return input, nil
|
||||
}
|
||||
if contract.SchemaFile != nil && !replaced {
|
||||
if def.SchemaFile == nil {
|
||||
return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile)
|
||||
}
|
||||
input.File = def.SchemaFile
|
||||
return input, nil
|
||||
}
|
||||
if contract.SchemaPath != "" {
|
||||
if def.SchemaRoot == nil {
|
||||
return validate.Input{}, errors.New("schema_path requires an applicable schema_root")
|
||||
}
|
||||
resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath)
|
||||
if err != nil {
|
||||
return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err)
|
||||
}
|
||||
input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved}
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func (r *Runner) structuredOutputFromValidationPlan(
|
||||
def *domain.PromptDefinition,
|
||||
contract domain.OutputContract,
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
@@ -246,7 +247,8 @@ type fakeValidator struct {
|
||||
validateCalls int
|
||||
}
|
||||
|
||||
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) {
|
||||
contract := input.Contract
|
||||
var schemaDocument any
|
||||
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||
f.schemaLoads++
|
||||
@@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
|
||||
staticArtifactReader{},
|
||||
staticRenderer{},
|
||||
scheduledClient,
|
||||
validate.NewStandardValidator("."),
|
||||
validate.NewValidator(),
|
||||
NewDefaultOutputRepairer(scheduledClient),
|
||||
admitter,
|
||||
)
|
||||
@@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
|
||||
DefaultProfile: "exec",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: mode,
|
||||
RepairAttempts: attempts,
|
||||
|
||||
51
internal/usecase/schema_binding_test.go
Normal file
51
internal/usecase/schema_binding_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
|
||||
fileName := "configured.json"
|
||||
def := &domain.PromptDefinition{
|
||||
ID: "collection/prompt",
|
||||
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"},
|
||||
SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"},
|
||||
Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName},
|
||||
}
|
||||
configured, err := bindValidationInput(def, def.Validation, false)
|
||||
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
|
||||
t.Fatalf("configured binding = %+v, %v", configured, err)
|
||||
}
|
||||
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
|
||||
contract, err := resolveOutputContract(context.Background(), def, inline)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := bindValidationInput(def, contract, true)
|
||||
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
|
||||
t.Fatalf("inline replacement = %+v, %v", selected, err)
|
||||
}
|
||||
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
|
||||
contract, err = resolveOutputContract(context.Background(), def, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err = bindValidationInput(def, contract, true)
|
||||
if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil {
|
||||
t.Fatalf("path replacement = %+v, %v", selected, err)
|
||||
}
|
||||
def.SchemaRoot = nil
|
||||
if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") {
|
||||
t.Fatalf("missing root = %v", err)
|
||||
}
|
||||
def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}
|
||||
contract.SchemaPath = "../outside.json"
|
||||
if _, err := bindValidationInput(def, contract, true); err == nil {
|
||||
t.Fatal("escaping path accepted")
|
||||
}
|
||||
}
|
||||
@@ -51,18 +51,17 @@ func prepareSmokeInferenceCase(
|
||||
),
|
||||
}, nil
|
||||
case domain.SmokeTestStructuredOutput:
|
||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
||||
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||
validator := validate.NewValidator()
|
||||
jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||
})
|
||||
}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||
}
|
||||
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||
plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||
Format: domain.FormatJSON,
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: smokeStructuredSchemaPath,
|
||||
})
|
||||
}, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
|
||||
|
||||
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
||||
|
||||
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||
v.state.validationCalls++
|
||||
return nil, errors.New("validation must not occur during target selection")
|
||||
}
|
||||
@@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any {
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
|
||||
"session_id": "{{.vars.session}}",
|
||||
"output": map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
||||
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||
},
|
||||
},
|
||||
map[string]any{"id": "backend.prompt",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||
"output": map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
||||
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user