Bind schema selections throughout execution
This commit is contained in:
@@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var underlays []configkit.Underlay
|
||||
var expectedMessage string
|
||||
|
||||
switch name {
|
||||
@@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||
}
|
||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
|
||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||
}, "."))
|
||||
}, ".")}})
|
||||
default:
|
||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||
}
|
||||
@@ -106,7 +107,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
"output": output,
|
||||
}},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
|
||||
@@ -245,19 +245,6 @@ func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
|
||||
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
|
||||
}
|
||||
|
||||
func TestPublicConstructionPreservesOptionFailureCause(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "missing.json")
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(path))
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid config preserving the missing-file cause", engine, err)
|
||||
}
|
||||
cause := errors.Unwrap(err)
|
||||
var pathErr *os.PathError
|
||||
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &pathErr) || pathErr.Path != path {
|
||||
t.Fatalf("construction cause = %v, want the original path error without a category branch", cause)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||
main := configkit.Input{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
|
||||
@@ -271,6 +271,8 @@ operation-local plan with no extra resource reads for `$ref` resolution.
|
||||
|
||||
## Stage 5 — Bind Effective Contracts In Every Execution Path
|
||||
|
||||
Status: complete.
|
||||
|
||||
Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and
|
||||
`smoke_inference_case.go`.
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
schemas := fstest.MapFS{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
@@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||
}},
|
||||
})},
|
||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
type deadlineValidation struct{ deadline time.Time }
|
||||
|
||||
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) {
|
||||
v.deadline, _ = ctx.Deadline()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
@@ -55,19 +55,19 @@ type recordingValidationPreparer struct {
|
||||
|
||||
type nilPlanPreparer struct{}
|
||||
|
||||
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (v *recordingValidationPreparer) PrepareValidation(
|
||||
_ context.Context,
|
||||
contract domain.OutputContract,
|
||||
input validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
v.prepareCalls++
|
||||
if v.prepareErr != nil {
|
||||
return nil, v.prepareErr
|
||||
}
|
||||
v.plan.contract = contract
|
||||
v.plan.contract = input.Contract
|
||||
return v.plan, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate(
|
||||
|
||||
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
|
||||
context.Context,
|
||||
domain.OutputContract,
|
||||
validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||
"go.maximumdirect.net/promptkit/internal/generation"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
@@ -73,6 +74,7 @@ type preparationState struct {
|
||||
effectiveModel domain.ExecutionTarget
|
||||
targetPresence domain.ExecutionTargetPresence
|
||||
effectiveContract domain.OutputContract
|
||||
validationInput validate.Input
|
||||
start time.Time
|
||||
transientRetries uint64
|
||||
}
|
||||
@@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation(
|
||||
promptDefinitionHash := promptSelection.hash
|
||||
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
|
||||
if err != nil {
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
return nil, contextErr
|
||||
}
|
||||
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err)
|
||||
}
|
||||
|
||||
selectedProfileID := req.ProfileID
|
||||
if strings.TrimSpace(selectedProfileID) == "" {
|
||||
@@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation(
|
||||
effectiveModel: effectiveModel,
|
||||
targetPresence: targetPresence,
|
||||
effectiveContract: effectiveContract,
|
||||
validationInput: validationInput,
|
||||
start: start,
|
||||
transientRetries: transientRetries,
|
||||
}, nil
|
||||
@@ -404,7 +414,7 @@ func (r *Runner) completePreparation(
|
||||
req domain.RunRequest,
|
||||
state *preparationState,
|
||||
) (*preparedOperation, error) {
|
||||
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||
validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -425,14 +435,15 @@ func (r *Runner) completePreparation(
|
||||
|
||||
func (r *Runner) prepareValidation(
|
||||
ctx context.Context,
|
||||
contract domain.OutputContract,
|
||||
promptID string,
|
||||
input validate.Input,
|
||||
) (validate.PreparedValidation, error) {
|
||||
if r.validator == nil {
|
||||
return noOpPreparedValidation{contract: contract}, nil
|
||||
return noOpPreparedValidation{contract: input.Contract}, nil
|
||||
}
|
||||
plan, err := r.validator.PrepareValidation(ctx, contract)
|
||||
plan, err := r.validator.PrepareValidation(ctx, input)
|
||||
if err != nil {
|
||||
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||
return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)}
|
||||
}
|
||||
if plan == nil {
|
||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||
@@ -440,6 +451,31 @@ func (r *Runner) prepareValidation(
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) {
|
||||
input := validate.Input{Contract: contract}
|
||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||
return input, nil
|
||||
}
|
||||
if contract.SchemaFile != nil && !replaced {
|
||||
if def.SchemaFile == nil {
|
||||
return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile)
|
||||
}
|
||||
input.File = def.SchemaFile
|
||||
return input, nil
|
||||
}
|
||||
if contract.SchemaPath != "" {
|
||||
if def.SchemaRoot == nil {
|
||||
return validate.Input{}, errors.New("schema_path requires an applicable schema_root")
|
||||
}
|
||||
resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath)
|
||||
if err != nil {
|
||||
return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err)
|
||||
}
|
||||
input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved}
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func (r *Runner) structuredOutputFromValidationPlan(
|
||||
def *domain.PromptDefinition,
|
||||
contract domain.OutputContract,
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
@@ -246,7 +247,8 @@ type fakeValidator struct {
|
||||
validateCalls int
|
||||
}
|
||||
|
||||
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) {
|
||||
contract := input.Contract
|
||||
var schemaDocument any
|
||||
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||
f.schemaLoads++
|
||||
@@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
|
||||
staticArtifactReader{},
|
||||
staticRenderer{},
|
||||
scheduledClient,
|
||||
validate.NewStandardValidator("."),
|
||||
validate.NewValidator(),
|
||||
NewDefaultOutputRepairer(scheduledClient),
|
||||
admitter,
|
||||
)
|
||||
@@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
|
||||
DefaultProfile: "exec",
|
||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."},
|
||||
Validation: domain.OutputContract{
|
||||
ValidationMode: mode,
|
||||
RepairAttempts: attempts,
|
||||
|
||||
51
internal/usecase/schema_binding_test.go
Normal file
51
internal/usecase/schema_binding_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
|
||||
fileName := "configured.json"
|
||||
def := &domain.PromptDefinition{
|
||||
ID: "collection/prompt",
|
||||
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"},
|
||||
SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"},
|
||||
Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName},
|
||||
}
|
||||
configured, err := bindValidationInput(def, def.Validation, false)
|
||||
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
|
||||
t.Fatalf("configured binding = %+v, %v", configured, err)
|
||||
}
|
||||
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
|
||||
contract, err := resolveOutputContract(context.Background(), def, inline)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := bindValidationInput(def, contract, true)
|
||||
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
|
||||
t.Fatalf("inline replacement = %+v, %v", selected, err)
|
||||
}
|
||||
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
|
||||
contract, err = resolveOutputContract(context.Background(), def, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err = bindValidationInput(def, contract, true)
|
||||
if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil {
|
||||
t.Fatalf("path replacement = %+v, %v", selected, err)
|
||||
}
|
||||
def.SchemaRoot = nil
|
||||
if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") {
|
||||
t.Fatalf("missing root = %v", err)
|
||||
}
|
||||
def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}
|
||||
contract.SchemaPath = "../outside.json"
|
||||
if _, err := bindValidationInput(def, contract, true); err == nil {
|
||||
t.Fatal("escaping path accepted")
|
||||
}
|
||||
}
|
||||
@@ -51,18 +51,17 @@ func prepareSmokeInferenceCase(
|
||||
),
|
||||
}, nil
|
||||
case domain.SmokeTestStructuredOutput:
|
||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
||||
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||
validator := validate.NewValidator()
|
||||
jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||
})
|
||||
}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||
}
|
||||
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||
plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
|
||||
Format: domain.FormatJSON,
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: smokeStructuredSchemaPath,
|
||||
})
|
||||
}, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
|
||||
|
||||
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
||||
|
||||
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||
func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
|
||||
v.state.validationCalls++
|
||||
return nil, errors.New("validation must not occur during target selection")
|
||||
}
|
||||
@@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any {
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
|
||||
"session_id": "{{.vars.session}}",
|
||||
"output": map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
||||
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||
},
|
||||
},
|
||||
map[string]any{"id": "backend.prompt",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||
"output": map[string]any{
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
|
||||
"format": "json", "validation_mode": "json_schema", "schema": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -3,7 +3,6 @@ package promptkit_test
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
@@ -52,8 +51,7 @@ func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) {
|
||||
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
|
||||
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
|
||||
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
|
||||
schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}}
|
||||
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, "."))
|
||||
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object"}}, false)
|
||||
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
|
||||
for name, prepared := range map[string]*promptkit.PreparedRun{
|
||||
"repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}),
|
||||
|
||||
@@ -242,7 +242,7 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
|
||||
func TestRunObservationValidationPreparationError(t *testing.T) {
|
||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||
var events []promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
||||
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
|
||||
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{
|
||||
Source: configkit.Mapping("requirements contract", map[string]any{
|
||||
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
|
||||
}),
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
@@ -12,7 +13,7 @@ import (
|
||||
|
||||
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
|
||||
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
|
||||
74
schema_execution_contract_test.go
Normal file
74
schema_execution_contract_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
|
||||
app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}}
|
||||
collection := fstest.MapFS{
|
||||
"collection/placeholder": {Data: []byte("context")},
|
||||
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
|
||||
}
|
||||
filesystem := configkit.NewFilesystem("collection", collection)
|
||||
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
|
||||
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"},
|
||||
}}}},
|
||||
}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"}
|
||||
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) {
|
||||
t.Fatalf("configured missing file = %v", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatal("preparation called model")
|
||||
}
|
||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
|
||||
result, err := engine.Run(context.Background(), base)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||
t.Fatalf("inline replacement = %+v, %v", result, err)
|
||||
}
|
||||
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
|
||||
t.Fatalf("inline submitted schema = %#v", got)
|
||||
}
|
||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
|
||||
if err := engine.ValidateRequest(context.Background(), base); err != nil {
|
||||
t.Fatalf("runtime path validation = %v", err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err = engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
|
||||
t.Fatalf("root replacement = %+v, %v", result, err)
|
||||
}
|
||||
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
|
||||
t.Fatalf("selected collection root schema = %#v", got)
|
||||
}
|
||||
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
|
||||
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("missing replacement selection = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
|
||||
engine := newSimpleEngine(t, "hello")
|
||||
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
|
||||
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("missing root = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -24,18 +24,17 @@ func (rejectWireSchemaLoad) Load(uri string) (any, error) {
|
||||
}
|
||||
|
||||
func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
||||
firstDir, secondDir := t.TempDir(), t.TempDir()
|
||||
first, second := filepath.Join(firstDir, "schema.json"), filepath.Join(secondDir, "schema.json")
|
||||
if err := os.WriteFile(first, []byte(`{"type":"string"}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondDir := t.TempDir()
|
||||
second := filepath.Join(secondDir, "schema.json")
|
||||
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||
})}
|
||||
filesystem := configkit.NewFilesystem("schema", os.DirFS(secondDir))
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||
var payload map[string]any
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||
@@ -43,7 +42,7 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(first), promptkit.WithSchemaFile(second))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -91,9 +90,11 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
|
||||
}
|
||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(path))
|
||||
filesystem := configkit.NewFilesystem("schema", os.DirFS(filepath.Dir(path)))
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -115,22 +116,19 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
|
||||
|
||||
func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, name := range []string{filepath.Join(dir, "missing.json"), dir} {
|
||||
if _, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(name)); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("source %q = %v", name, err)
|
||||
}
|
||||
}
|
||||
root := filepath.Join(dir, "schema.json")
|
||||
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
|
||||
})}
|
||||
filesystem := configkit.NewFilesystem("schema", os.DirFS(dir))
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||
calls := 0
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(root))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
@@ -308,7 +309,10 @@ type failingSchemaFS struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (source *failingSchemaFS) Open(string) (fs.File, error) {
|
||||
func (source *failingSchemaFS) Open(name string) (fs.File, error) {
|
||||
if name == "." {
|
||||
return (fstest.MapFS{}).Open(name)
|
||||
}
|
||||
source.mu.Lock()
|
||||
source.calls++
|
||||
source.mu.Unlock()
|
||||
@@ -346,8 +350,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
|
||||
},
|
||||
}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}},
|
||||
promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas"))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemaSource, "schemas")}}}}},
|
||||
promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
@@ -146,7 +145,6 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
|
||||
"schema": onePrompt("schema", schemaPrompt)["schema"],
|
||||
}
|
||||
engine := requirementsEngine(t, prompts, profiles,
|
||||
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user