Bind schema selections throughout execution
This commit is contained in:
@@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||||
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||||
|
var underlays []configkit.Underlay
|
||||||
var expectedMessage string
|
var expectedMessage string
|
||||||
|
|
||||||
switch name {
|
switch name {
|
||||||
@@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||||
}
|
}
|
||||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
|
||||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||||
}, "."))
|
}, ".")}})
|
||||||
default:
|
default:
|
||||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||||
}
|
}
|
||||||
@@ -106,7 +107,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
"output": output,
|
"output": output,
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -245,19 +245,6 @@ func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
|
|||||||
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
|
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublicConstructionPreservesOptionFailureCause(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "missing.json")
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(path))
|
|
||||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatalf("construction = (%#v, %v), want invalid config preserving the missing-file cause", engine, err)
|
|
||||||
}
|
|
||||||
cause := errors.Unwrap(err)
|
|
||||||
var pathErr *os.PathError
|
|
||||||
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &pathErr) || pathErr.Path != path {
|
|
||||||
t.Fatalf("construction cause = %v, want the original path error without a category branch", cause)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||||
main := configkit.Input{}
|
main := configkit.Input{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||||
|
|||||||
@@ -271,6 +271,8 @@ operation-local plan with no extra resource reads for `$ref` resolution.
|
|||||||
|
|
||||||
## Stage 5 — Bind Effective Contracts In Every Execution Path
|
## Stage 5 — Bind Effective Contracts In Every Execution Path
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and
|
Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and
|
||||||
`smoke_inference_case.go`.
|
`smoke_inference_case.go`.
|
||||||
|
|
||||||
|
|||||||
@@ -144,7 +144,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
|||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
schemas := fstest.MapFS{}
|
schemas := fstest.MapFS{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||||
@@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
|||||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||||
}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
|
|
||||||
type deadlineValidation struct{ deadline time.Time }
|
type deadlineValidation struct{ deadline time.Time }
|
||||||
|
|
||||||
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) {
|
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) {
|
||||||
v.deadline, _ = ctx.Deadline()
|
v.deadline, _ = ctx.Deadline()
|
||||||
return v, nil
|
return v, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,19 +55,19 @@ type recordingValidationPreparer struct {
|
|||||||
|
|
||||||
type nilPlanPreparer struct{}
|
type nilPlanPreparer struct{}
|
||||||
|
|
||||||
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) PrepareValidation(
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
contract domain.OutputContract,
|
input validate.Input,
|
||||||
) (validate.PreparedValidation, error) {
|
) (validate.PreparedValidation, error) {
|
||||||
v.prepareCalls++
|
v.prepareCalls++
|
||||||
if v.prepareErr != nil {
|
if v.prepareErr != nil {
|
||||||
return nil, v.prepareErr
|
return nil, v.prepareErr
|
||||||
}
|
}
|
||||||
v.plan.contract = contract
|
v.plan.contract = input.Contract
|
||||||
return v.plan, nil
|
return v.plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate(
|
|||||||
|
|
||||||
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||||
context.Context,
|
context.Context,
|
||||||
domain.OutputContract,
|
validate.Input,
|
||||||
) (validate.PreparedValidation, error) {
|
) (validate.PreparedValidation, error) {
|
||||||
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||||
"go.maximumdirect.net/promptkit/internal/generation"
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
@@ -73,6 +74,7 @@ type preparationState struct {
|
|||||||
effectiveModel domain.ExecutionTarget
|
effectiveModel domain.ExecutionTarget
|
||||||
targetPresence domain.ExecutionTargetPresence
|
targetPresence domain.ExecutionTargetPresence
|
||||||
effectiveContract domain.OutputContract
|
effectiveContract domain.OutputContract
|
||||||
|
validationInput validate.Input
|
||||||
start time.Time
|
start time.Time
|
||||||
transientRetries uint64
|
transientRetries uint64
|
||||||
}
|
}
|
||||||
@@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation(
|
|||||||
promptDefinitionHash := promptSelection.hash
|
promptDefinitionHash := promptSelection.hash
|
||||||
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
|
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if contextErr := ctx.Err(); contextErr != nil {
|
||||||
|
return nil, contextErr
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
selectedProfileID := req.ProfileID
|
selectedProfileID := req.ProfileID
|
||||||
if strings.TrimSpace(selectedProfileID) == "" {
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
@@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation(
|
|||||||
effectiveModel: effectiveModel,
|
effectiveModel: effectiveModel,
|
||||||
targetPresence: targetPresence,
|
targetPresence: targetPresence,
|
||||||
effectiveContract: effectiveContract,
|
effectiveContract: effectiveContract,
|
||||||
|
validationInput: validationInput,
|
||||||
start: start,
|
start: start,
|
||||||
transientRetries: transientRetries,
|
transientRetries: transientRetries,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -404,7 +414,7 @@ func (r *Runner) completePreparation(
|
|||||||
req domain.RunRequest,
|
req domain.RunRequest,
|
||||||
state *preparationState,
|
state *preparationState,
|
||||||
) (*preparedOperation, error) {
|
) (*preparedOperation, error) {
|
||||||
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -425,14 +435,15 @@ func (r *Runner) completePreparation(
|
|||||||
|
|
||||||
func (r *Runner) prepareValidation(
|
func (r *Runner) prepareValidation(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
contract domain.OutputContract,
|
promptID string,
|
||||||
|
input validate.Input,
|
||||||
) (validate.PreparedValidation, error) {
|
) (validate.PreparedValidation, error) {
|
||||||
if r.validator == nil {
|
if r.validator == nil {
|
||||||
return noOpPreparedValidation{contract: contract}, nil
|
return noOpPreparedValidation{contract: input.Contract}, nil
|
||||||
}
|
}
|
||||||
plan, err := r.validator.PrepareValidation(ctx, contract)
|
plan, err := r.validator.PrepareValidation(ctx, input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)}
|
||||||
}
|
}
|
||||||
if plan == nil {
|
if plan == nil {
|
||||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
@@ -440,6 +451,31 @@ func (r *Runner) prepareValidation(
|
|||||||
return plan, nil
|
return plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) {
|
||||||
|
input := validate.Input{Contract: contract}
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
if contract.SchemaFile != nil && !replaced {
|
||||||
|
if def.SchemaFile == nil {
|
||||||
|
return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile)
|
||||||
|
}
|
||||||
|
input.File = def.SchemaFile
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
if contract.SchemaPath != "" {
|
||||||
|
if def.SchemaRoot == nil {
|
||||||
|
return validate.Input{}, errors.New("schema_path requires an applicable schema_root")
|
||||||
|
}
|
||||||
|
resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err)
|
||||||
|
}
|
||||||
|
input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved}
|
||||||
|
}
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) structuredOutputFromValidationPlan(
|
func (r *Runner) structuredOutputFromValidationPlan(
|
||||||
def *domain.PromptDefinition,
|
def *domain.PromptDefinition,
|
||||||
contract domain.OutputContract,
|
contract domain.OutputContract,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -246,7 +247,8 @@ type fakeValidator struct {
|
|||||||
validateCalls int
|
validateCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) {
|
||||||
|
contract := input.Contract
|
||||||
var schemaDocument any
|
var schemaDocument any
|
||||||
if contract.ValidationMode == domain.ValidationJSONSchema {
|
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||||
f.schemaLoads++
|
f.schemaLoads++
|
||||||
@@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
|
|||||||
staticArtifactReader{},
|
staticArtifactReader{},
|
||||||
staticRenderer{},
|
staticRenderer{},
|
||||||
scheduledClient,
|
scheduledClient,
|
||||||
validate.NewStandardValidator("."),
|
validate.NewValidator(),
|
||||||
NewDefaultOutputRepairer(scheduledClient),
|
NewDefaultOutputRepairer(scheduledClient),
|
||||||
admitter,
|
admitter,
|
||||||
)
|
)
|
||||||
@@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
|
|||||||
DefaultProfile: "exec",
|
DefaultProfile: "exec",
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||||
|
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."},
|
||||||
Validation: domain.OutputContract{
|
Validation: domain.OutputContract{
|
||||||
ValidationMode: mode,
|
ValidationMode: mode,
|
||||||
RepairAttempts: attempts,
|
RepairAttempts: attempts,
|
||||||
|
|||||||
51
internal/usecase/schema_binding_test.go
Normal file
51
internal/usecase/schema_binding_test.go
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
|
||||||
|
fileName := "configured.json"
|
||||||
|
def := &domain.PromptDefinition{
|
||||||
|
ID: "collection/prompt",
|
||||||
|
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"},
|
||||||
|
SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"},
|
||||||
|
Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName},
|
||||||
|
}
|
||||||
|
configured, err := bindValidationInput(def, def.Validation, false)
|
||||||
|
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
|
||||||
|
t.Fatalf("configured binding = %+v, %v", configured, err)
|
||||||
|
}
|
||||||
|
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
|
||||||
|
contract, err := resolveOutputContract(context.Background(), def, inline)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
selected, err := bindValidationInput(def, contract, true)
|
||||||
|
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
|
||||||
|
t.Fatalf("inline replacement = %+v, %v", selected, err)
|
||||||
|
}
|
||||||
|
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
|
||||||
|
contract, err = resolveOutputContract(context.Background(), def, path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
selected, err = bindValidationInput(def, contract, true)
|
||||||
|
if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil {
|
||||||
|
t.Fatalf("path replacement = %+v, %v", selected, err)
|
||||||
|
}
|
||||||
|
def.SchemaRoot = nil
|
||||||
|
if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") {
|
||||||
|
t.Fatalf("missing root = %v", err)
|
||||||
|
}
|
||||||
|
def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}
|
||||||
|
contract.SchemaPath = "../outside.json"
|
||||||
|
if _, err := bindValidationInput(def, contract, true); err == nil {
|
||||||
|
t.Fatal("escaping path accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -51,18 +51,17 @@ func prepareSmokeInferenceCase(
|
|||||||
),
|
),
|
||||||
}, nil
|
}, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
validator := validate.NewValidator()
|
||||||
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||||
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||||
})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
SchemaPath: smokeStructuredSchemaPath,
|
}, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}})
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
|
|||||||
|
|
||||||
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
||||||
|
|
||||||
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||||
v.state.validationCalls++
|
v.state.validationCalls++
|
||||||
return nil, errors.New("validation must not occur during target selection")
|
return nil, errors.New("validation must not occur during target selection")
|
||||||
}
|
}
|
||||||
@@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any {
|
|||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
|
||||||
"session_id": "{{.vars.session}}",
|
"session_id": "{{.vars.session}}",
|
||||||
"output": map[string]any{
|
"output": map[string]any{
|
||||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
map[string]any{"id": "backend.prompt",
|
map[string]any{"id": "backend.prompt",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
"output": map[string]any{
|
"output": map[string]any{
|
||||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package promptkit_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
@@ -52,8 +51,7 @@ func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) {
|
|||||||
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
|
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
|
||||||
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
|
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
|
||||||
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
|
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
|
||||||
schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}}
|
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object"}}, false)
|
||||||
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, "."))
|
|
||||||
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
|
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
|
||||||
for name, prepared := range map[string]*promptkit.PreparedRun{
|
for name, prepared := range map[string]*promptkit.PreparedRun{
|
||||||
"repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}),
|
"repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}),
|
||||||
|
|||||||
@@ -242,7 +242,7 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
|
|||||||
func TestRunObservationValidationPreparationError(t *testing.T) {
|
func TestRunObservationValidationPreparationError(t *testing.T) {
|
||||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||||
var events []promptkit.ExecutionEvent
|
var events []promptkit.ExecutionEvent
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||||
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
||||||
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("requirements contract", map[string]any{
|
Source: configkit.Mapping("requirements contract", map[string]any{
|
||||||
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
|
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
@@ -12,7 +13,7 @@ import (
|
|||||||
|
|
||||||
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
||||||
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
|||||||
74
schema_execution_contract_test.go
Normal file
74
schema_execution_contract_test.go
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
||||||
|
app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}}
|
||||||
|
collection := fstest.MapFS{
|
||||||
|
"collection/placeholder": {Data: []byte("context")},
|
||||||
|
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
|
||||||
|
}
|
||||||
|
filesystem := configkit.NewFilesystem("collection", collection)
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
|
||||||
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"},
|
||||||
|
}}}},
|
||||||
|
}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"}
|
||||||
|
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) {
|
||||||
|
t.Fatalf("configured missing file = %v", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatal("preparation called model")
|
||||||
|
}
|
||||||
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
|
||||||
|
result, err := engine.Run(context.Background(), base)
|
||||||
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
t.Fatalf("inline replacement = %+v, %v", result, err)
|
||||||
|
}
|
||||||
|
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
|
||||||
|
t.Fatalf("inline submitted schema = %#v", got)
|
||||||
|
}
|
||||||
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
||||||
|
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
||||||
|
t.Fatalf("runtime path validation = %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err = engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
t.Fatalf("root replacement = %+v, %v", result, err)
|
||||||
|
}
|
||||||
|
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
|
||||||
|
t.Fatalf("selected collection root schema = %#v", got)
|
||||||
|
}
|
||||||
|
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
|
||||||
|
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing replacement selection = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "hello")
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
|
||||||
|
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing root = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,18 +24,17 @@ func (rejectWireSchemaLoad) Load(uri string) (any, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
||||||
firstDir, secondDir := t.TempDir(), t.TempDir()
|
secondDir := t.TempDir()
|
||||||
first, second := filepath.Join(firstDir, "schema.json"), filepath.Join(secondDir, "schema.json")
|
second := filepath.Join(secondDir, "schema.json")
|
||||||
if err := os.WriteFile(first, []byte(`{"type":"string"}`), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
|
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
|
filesystem := configkit.NewFilesystem("schema", os.DirFS(secondDir))
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||||
var payload map[string]any
|
var payload map[string]any
|
||||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||||
@@ -43,7 +42,7 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(first), promptkit.WithSchemaFile(second))
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -91,9 +90,11 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(path))
|
filesystem := configkit.NewFilesystem("schema", os.DirFS(filepath.Dir(path)))
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -115,22 +116,19 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
|
|||||||
|
|
||||||
func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) {
|
func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
for _, name := range []string{filepath.Join(dir, "missing.json"), dir} {
|
|
||||||
if _, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(name)); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
||||||
t.Fatalf("source %q = %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
root := filepath.Join(dir, "schema.json")
|
root := filepath.Join(dir, "schema.json")
|
||||||
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
|
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||||
})}
|
})}
|
||||||
|
filesystem := configkit.NewFilesystem("schema", os.DirFS(dir))
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||||
calls := 0
|
calls := 0
|
||||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })}
|
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(root))
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
@@ -308,7 +309,10 @@ type failingSchemaFS struct {
|
|||||||
calls int
|
calls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (source *failingSchemaFS) Open(string) (fs.File, error) {
|
func (source *failingSchemaFS) Open(name string) (fs.File, error) {
|
||||||
|
if name == "." {
|
||||||
|
return (fstest.MapFS{}).Open(name)
|
||||||
|
}
|
||||||
source.mu.Lock()
|
source.mu.Lock()
|
||||||
source.calls++
|
source.calls++
|
||||||
source.mu.Unlock()
|
source.mu.Unlock()
|
||||||
@@ -346,8 +350,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
|||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
})}
|
})}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}},
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemaSource, "schemas")}}}}},
|
||||||
promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas"))
|
promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
@@ -146,7 +145,6 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
|
|||||||
"schema": onePrompt("schema", schemaPrompt)["schema"],
|
"schema": onePrompt("schema", schemaPrompt)["schema"],
|
||||||
}
|
}
|
||||||
engine := requirementsEngine(t, prompts, profiles,
|
engine := requirementsEngine(t, prompts, profiles,
|
||||||
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
|
|
||||||
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user