From 6fbbaa9cfa0c9b1dfd308826aae13eb3782ecfda Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 15:36:44 +0000 Subject: [PATCH] Bind schema selections throughout execution --- assembled_workflow_benchmark_test.go | 7 +- construction_contract_test.go | 13 ---- docs/roadmap/implementation.md | 2 + frozen_rendering_contract_test.go | 4 +- internal/usecase/operation_budget_test.go | 2 +- internal/usecase/prepared_execution_test.go | 6 +- .../prepared_snapshot_benchmark_test.go | 2 +- internal/usecase/runner.go | 46 ++++++++++-- internal/usecase/runner_test.go | 7 +- internal/usecase/schema_binding_test.go | 51 +++++++++++++ internal/usecase/smoke_inference_case.go | 11 ++- internal/usecase/smoke_selection_test.go | 6 +- message_content_hash_contract_test.go | 4 +- observation_contract_test.go | 2 +- requirements_inspection_contract_test.go | 2 +- resource_isolation_test.go | 3 +- schema_execution_contract_test.go | 74 +++++++++++++++++++ schema_file_contract_test.go | 30 ++++---- smoke_integration_contract_test.go | 10 ++- validate_request_contract_test.go | 2 - 20 files changed, 218 insertions(+), 66 deletions(-) create mode 100644 internal/usecase/schema_binding_test.go create mode 100644 schema_execution_contract_test.go diff --git a/assembled_workflow_benchmark_test.go b/assembled_workflow_benchmark_test.go index 623de43..940c8c4 100644 --- a/assembled_workflow_benchmark_test.go +++ b/assembled_workflow_benchmark_test.go @@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix output := map[string]any{"format": "text", "validation_mode": "none"} profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"} options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})} + var underlays []configkit.Underlay var expectedMessage string switch name { @@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix "format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json", } options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response}) - options = append(options, promptkit.WithSchemaFS(fstest.MapFS{ + underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{ "workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)}, - }, ".")) + }, ".")}}) default: b.Fatalf("unknown assembled workflow fixture %q", name) } @@ -106,7 +107,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix "output": output, }}, } - engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{ Source: configkit.Mapping("assembled workflow benchmark", config), }}}, options...) if err != nil { diff --git a/construction_contract_test.go b/construction_contract_test.go index b11eb1b..ae7433c 100644 --- a/construction_contract_test.go +++ b/construction_contract_test.go @@ -245,19 +245,6 @@ func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) { t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics()) } -func TestPublicConstructionPreservesOptionFailureCause(t *testing.T) { - path := filepath.Join(t.TempDir(), "missing.json") - engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(path)) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.Is(err, os.ErrNotExist) { - t.Fatalf("construction = (%#v, %v), want invalid config preserving the missing-file cause", engine, err) - } - cause := errors.Unwrap(err) - var pathErr *os.PathError - if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &pathErr) || pathErr.Path != path { - t.Fatalf("construction cause = %v, want the original path error without a category branch", cause) - } -} - func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) { main := configkit.Input{} engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}}) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fdb63c3..5877828 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -271,6 +271,8 @@ operation-local plan with no extra resource reads for `$ref` resolution. ## Stage 5 — Bind Effective Contracts In Every Execution Path +Status: complete. + Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and `smoke_inference_case.go`. diff --git a/frozen_rendering_contract_test.go b/frozen_rendering_contract_test.go index 3eaf1cf..a05848d 100644 --- a/frozen_rendering_contract_test.go +++ b/frozen_rendering_contract_test.go @@ -144,7 +144,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes endpoint, model := "https://example.invalid/v1", "offline-model" schemas := fstest.MapFS{} engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{ {ID: "offline", Endpoint: &endpoint, Model: &model}, })}}}}, Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{ @@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, }}, })}, - }}, promptkit.WithSchemaFS(schemas, ".")) + }}) if err != nil { t.Fatalf("construct engine before request resources exist: %v", err) } diff --git a/internal/usecase/operation_budget_test.go b/internal/usecase/operation_budget_test.go index b16a50c..5496815 100644 --- a/internal/usecase/operation_budget_test.go +++ b/internal/usecase/operation_budget_test.go @@ -13,7 +13,7 @@ import ( type deadlineValidation struct{ deadline time.Time } -func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) { +func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) { v.deadline, _ = ctx.Deadline() return v, nil } diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index 4b5ac19..152c6bc 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -55,19 +55,19 @@ type recordingValidationPreparer struct { type nilPlanPreparer struct{} -func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) { +func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) { return nil, nil } func (v *recordingValidationPreparer) PrepareValidation( _ context.Context, - contract domain.OutputContract, + input validate.Input, ) (validate.PreparedValidation, error) { v.prepareCalls++ if v.prepareErr != nil { return nil, v.prepareErr } - v.plan.contract = contract + v.plan.contract = input.Contract return v.plan, nil } diff --git a/internal/usecase/prepared_snapshot_benchmark_test.go b/internal/usecase/prepared_snapshot_benchmark_test.go index 243af50..a673e2e 100644 --- a/internal/usecase/prepared_snapshot_benchmark_test.go +++ b/internal/usecase/prepared_snapshot_benchmark_test.go @@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate( func (validator preparedSnapshotBenchmarkValidator) PrepareValidation( context.Context, - domain.OutputContract, + validate.Input, ) (validate.PreparedValidation, error) { return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 8bb8d65..90e19dd 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -18,6 +18,7 @@ import ( "go.maximumdirect.net/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/domain" + "go.maximumdirect.net/promptkit/internal/filecatalog" "go.maximumdirect.net/promptkit/internal/generation" "go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/llm" @@ -73,6 +74,7 @@ type preparationState struct { effectiveModel domain.ExecutionTarget targetPresence domain.ExecutionTargetPresence effectiveContract domain.OutputContract + validationInput validate.Input start time.Time transientRetries uint64 } @@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation( promptDefinitionHash := promptSelection.hash effectiveContract, err := resolveOutputContract(ctx, def, req.Validation) if err != nil { + if contextErr := ctx.Err(); contextErr != nil { + return nil, contextErr + } return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err) } + validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil) + if err != nil { + return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err) + } selectedProfileID := req.ProfileID if strings.TrimSpace(selectedProfileID) == "" { @@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation( effectiveModel: effectiveModel, targetPresence: targetPresence, effectiveContract: effectiveContract, + validationInput: validationInput, start: start, transientRetries: transientRetries, }, nil @@ -404,7 +414,7 @@ func (r *Runner) completePreparation( req domain.RunRequest, state *preparationState, ) (*preparedOperation, error) { - validationPlan, err := r.prepareValidation(ctx, state.effectiveContract) + validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput) if err != nil { return nil, err } @@ -425,14 +435,15 @@ func (r *Runner) completePreparation( func (r *Runner) prepareValidation( ctx context.Context, - contract domain.OutputContract, + promptID string, + input validate.Input, ) (validate.PreparedValidation, error) { if r.validator == nil { - return noOpPreparedValidation{contract: contract}, nil + return noOpPreparedValidation{contract: input.Contract}, nil } - plan, err := r.validator.PrepareValidation(ctx, contract) + plan, err := r.validator.PrepareValidation(ctx, input) if err != nil { - return nil, &OperationError{Category: ErrValidation, Cause: err} + return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)} } if plan == nil { return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation) @@ -440,6 +451,31 @@ func (r *Runner) prepareValidation( return plan, nil } +func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) { + input := validate.Input{Contract: contract} + if contract.ValidationMode != domain.ValidationJSONSchema { + return input, nil + } + if contract.SchemaFile != nil && !replaced { + if def.SchemaFile == nil { + return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile) + } + input.File = def.SchemaFile + return input, nil + } + if contract.SchemaPath != "" { + if def.SchemaRoot == nil { + return validate.Input{}, errors.New("schema_path requires an applicable schema_root") + } + resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath) + if err != nil { + return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err) + } + input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved} + } + return input, nil +} + func (r *Runner) structuredOutputFromValidationPlan( def *domain.PromptDefinition, contract domain.OutputContract, diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index c1e37f8..17f7ba3 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "testing" + "testing/fstest" "text/template" "time" @@ -246,7 +247,8 @@ type fakeValidator struct { validateCalls int } -func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) { +func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) { + contract := input.Contract var schemaDocument any if contract.ValidationMode == domain.ValidationJSONSchema { f.schemaLoads++ @@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi staticArtifactReader{}, staticRenderer{}, scheduledClient, - validate.NewStandardValidator("."), + validate.NewValidator(), NewDefaultOutputRepairer(scheduledClient), admitter, ) @@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts DefaultProfile: "exec", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, + SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."}, Validation: domain.OutputContract{ ValidationMode: mode, RepairAttempts: attempts, diff --git a/internal/usecase/schema_binding_test.go b/internal/usecase/schema_binding_test.go new file mode 100644 index 0000000..0795497 --- /dev/null +++ b/internal/usecase/schema_binding_test.go @@ -0,0 +1,51 @@ +package usecase + +import ( + "context" + "strings" + "testing" + "testing/fstest" + + "go.maximumdirect.net/promptkit/internal/domain" +) + +func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) { + fileName := "configured.json" + def := &domain.PromptDefinition{ + ID: "collection/prompt", + SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}, + SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"}, + Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName}, + } + configured, err := bindValidationInput(def, def.Validation, false) + if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" { + t.Fatalf("configured binding = %+v, %v", configured, err) + } + inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false} + contract, err := resolveOutputContract(context.Background(), def, inline) + if err != nil { + t.Fatal(err) + } + selected, err := bindValidationInput(def, contract, true) + if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil { + t.Fatalf("inline replacement = %+v, %v", selected, err) + } + path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"} + contract, err = resolveOutputContract(context.Background(), def, path) + if err != nil { + t.Fatal(err) + } + selected, err = bindValidationInput(def, contract, true) + if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil { + t.Fatalf("path replacement = %+v, %v", selected, err) + } + def.SchemaRoot = nil + if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") { + t.Fatalf("missing root = %v", err) + } + def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"} + contract.SchemaPath = "../outside.json" + if _, err := bindValidationInput(def, contract, true); err == nil { + t.Fatal("escaping path accepted") + } +} diff --git a/internal/usecase/smoke_inference_case.go b/internal/usecase/smoke_inference_case.go index d3aabda..2a55375 100644 --- a/internal/usecase/smoke_inference_case.go +++ b/internal/usecase/smoke_inference_case.go @@ -51,18 +51,17 @@ func prepareSmokeInferenceCase( ), }, nil case domain.SmokeTestStructuredOutput: - validator := validate.NewFSValidator(smokeSchemaFS, ".") - jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{ + validator := validate.NewValidator() + jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{ Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, - }) + }}) if err != nil { return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err) } - plan, err := validator.PrepareValidation(ctx, domain.OutputContract{ + plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{ Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, - SchemaPath: smokeStructuredSchemaPath, - }) + }, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}}) if err != nil { return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err) } diff --git a/internal/usecase/smoke_selection_test.go b/internal/usecase/smoke_selection_test.go index 17c66a8..1332ed2 100644 --- a/internal/usecase/smoke_selection_test.go +++ b/internal/usecase/smoke_selection_test.go @@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do type selectionTestValidator struct{ state *selectionTestCollaborators } -func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) { +func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) { v.state.validationCalls++ return nil, errors.New("validation must not occur during target selection") } @@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any { "messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}}, "session_id": "{{.vars.session}}", "output": map[string]any{ - "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", + "format": "json", "validation_mode": "json_schema", "schema": false, }, }, map[string]any{"id": "backend.prompt", "messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "output": map[string]any{ - "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", + "format": "json", "validation_mode": "json_schema", "schema": false, }, }, }, diff --git a/message_content_hash_contract_test.go b/message_content_hash_contract_test.go index 1f8af8f..e2da8aa 100644 --- a/message_content_hash_contract_test.go +++ b/message_content_hash_contract_test.go @@ -3,7 +3,6 @@ package promptkit_test import ( "context" "testing" - "testing/fstest" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" @@ -52,8 +51,7 @@ func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) { session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"}) cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true) cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"}) - schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}} - schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, ".")) + schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object"}}, false) schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"}) for name, prepared := range map[string]*promptkit.PreparedRun{ "repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}), diff --git a/observation_contract_test.go b/observation_contract_test.go index 0d6d09a..44ce072 100644 --- a/observation_contract_test.go +++ b/observation_contract_test.go @@ -242,7 +242,7 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) { func TestRunObservationValidationPreparationError(t *testing.T) { engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}) var events []promptkit.ExecutionEvent - result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 { t.Fatalf("result=%+v error=%v events=%+v", result, err, events) } diff --git a/requirements_inspection_contract_test.go b/requirements_inspection_contract_test.go index 5f688a7..8ce01f0 100644 --- a/requirements_inspection_contract_test.go +++ b/requirements_inspection_contract_test.go @@ -16,7 +16,7 @@ import ( func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine { t.Helper() - engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{ Source: configkit.Mapping("requirements contract", map[string]any{ "prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles), }), diff --git a/resource_isolation_test.go b/resource_isolation_test.go index 3729eed..cd43404 100644 --- a/resource_isolation_test.go +++ b/resource_isolation_test.go @@ -5,6 +5,7 @@ import ( "strings" "sync" "testing" + "testing/fstest" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" @@ -12,7 +13,7 @@ import ( func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} - engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{ + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{ "backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}}, "profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}}, "prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, diff --git a/schema_execution_contract_test.go b/schema_execution_contract_test.go new file mode 100644 index 0000000..8b5069a --- /dev/null +++ b/schema_execution_contract_test.go @@ -0,0 +1,74 @@ +package promptkit_test + +import ( + "context" + "errors" + "testing" + "testing/fstest" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) { + app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}} + collection := fstest.MapFS{ + "collection/placeholder": {Data: []byte("context")}, + "collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)}, + } + filesystem := configkit.NewFilesystem("collection", collection) + input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{ + "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, + "collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{ + "id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"}, + }}}}, + }), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}} + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"} + if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) { + t.Fatalf("configured missing file = %v", err) + } + if len(client.requests) != 0 { + t.Fatal("preparation called model") + } + base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true} + result, err := engine.Run(context.Background(), base) + if err != nil || result.Validation.Status != promptkit.ValidationPassed { + t.Fatalf("inline replacement = %+v, %v", result, err) + } + if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true { + t.Fatalf("inline submitted schema = %#v", got) + } + base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"} + if err := engine.ValidateRequest(context.Background(), base); err != nil { + t.Fatalf("runtime path validation = %v", err) + } + prepared, err := engine.PrepareExecution(context.Background(), base) + if err != nil { + t.Fatal(err) + } + result, err = engine.RunPrepared(context.Background(), prepared) + if err != nil || result.Validation.Status != promptkit.ValidationPassed { + t.Fatalf("root replacement = %+v, %v", result, err) + } + if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" { + t.Fatalf("selected collection root schema = %#v", got) + } + base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema} + if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("missing replacement selection = %v", err) + } +} + +func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) { + engine := newSimpleEngine(t, "hello") + request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}} + if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("missing root = %v", err) + } +} diff --git a/schema_file_contract_test.go b/schema_file_contract_test.go index 29a00f1..6082982 100644 --- a/schema_file_contract_test.go +++ b/schema_file_contract_test.go @@ -24,18 +24,17 @@ func (rejectWireSchemaLoad) Load(uri string) (any, error) { } func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) { - firstDir, secondDir := t.TempDir(), t.TempDir() - first, second := filepath.Join(firstDir, "schema.json"), filepath.Join(secondDir, "schema.json") - if err := os.WriteFile(first, []byte(`{"type":"string"}`), 0600); err != nil { - t.Fatal(err) - } + secondDir := t.TempDir() + second := filepath.Join(secondDir, "schema.json") if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil { t.Fatal(err) } input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, - "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}}, })} + filesystem := configkit.NewFilesystem("schema", os.DirFS(secondDir)) + input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."} var payload map[string]any client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { @@ -43,7 +42,7 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) { } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil })} - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(first), promptkit.WithSchemaFile(second)) + engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}) if err != nil { t.Fatal(err) } @@ -91,9 +90,11 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) { } input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, - "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}}, })} - engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(path)) + filesystem := configkit.NewFilesystem("schema", os.DirFS(filepath.Dir(path))) + input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}) if err != nil { t.Fatal(err) } @@ -115,22 +116,19 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) { func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) { dir := t.TempDir() - for _, name := range []string{filepath.Join(dir, "missing.json"), dir} { - if _, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(name)); !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("source %q = %v", name, err) - } - } root := filepath.Join(dir, "schema.json") if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil { t.Fatal(err) } input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, - "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}}, })} + filesystem := configkit.NewFilesystem("schema", os.DirFS(dir)) + input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."} calls := 0 client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })} - engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(root)) + engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}) if err != nil { t.Fatal(err) } diff --git a/smoke_integration_contract_test.go b/smoke_integration_contract_test.go index 55a0610..dacd7fc 100644 --- a/smoke_integration_contract_test.go +++ b/smoke_integration_contract_test.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "testing" + "testing/fstest" "time" "go.maximumdirect.net/configkit" @@ -308,7 +309,10 @@ type failingSchemaFS struct { calls int } -func (source *failingSchemaFS) Open(string) (fs.File, error) { +func (source *failingSchemaFS) Open(name string) (fs.File, error) { + if name == "." { + return (fstest.MapFS{}).Open(name) + } source.mu.Lock() source.calls++ source.mu.Unlock() @@ -346,8 +350,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange }, }}, })} - engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, - promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas")) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemaSource, "schemas")}}}}}, + promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader)) if err != nil { t.Fatalf("construct engine: %v", err) } diff --git a/validate_request_contract_test.go b/validate_request_contract_test.go index 6001f57..e2da687 100644 --- a/validate_request_contract_test.go +++ b/validate_request_contract_test.go @@ -6,7 +6,6 @@ import ( "path/filepath" "sync/atomic" "testing" - "testing/fstest" "go.maximumdirect.net/promptkit" ) @@ -146,7 +145,6 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) { "schema": onePrompt("schema", schemaPrompt)["schema"], } engine := requirementsEngine(t, prompts, profiles, - promptkit.WithSchemaFS(fstest.MapFS{}, "."), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}), )