Bind schema selections throughout execution

This commit is contained in:
2026-11-20 15:36:44 +00:00
parent 2d96dfa696
commit 6fbbaa9cfa
20 changed files with 218 additions and 66 deletions

View File

@@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
output := map[string]any{"format": "text", "validation_mode": "none"} output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"} profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})} options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var underlays []configkit.Underlay
var expectedMessage string var expectedMessage string
switch name { switch name {
@@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json", "format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
} }
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response}) options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{ underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)}, "workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
}, ".")) }, ".")}})
default: default:
b.Fatalf("unknown assembled workflow fixture %q", name) b.Fatalf("unknown assembled workflow fixture %q", name)
} }
@@ -106,7 +107,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
"output": output, "output": output,
}}, }},
} }
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
Source: configkit.Mapping("assembled workflow benchmark", config), Source: configkit.Mapping("assembled workflow benchmark", config),
}}}, options...) }}}, options...)
if err != nil { if err != nil {

View File

@@ -245,19 +245,6 @@ func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics()) t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
} }
func TestPublicConstructionPreservesOptionFailureCause(t *testing.T) {
path := filepath.Join(t.TempDir(), "missing.json")
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(path))
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.Is(err, os.ErrNotExist) {
t.Fatalf("construction = (%#v, %v), want invalid config preserving the missing-file cause", engine, err)
}
cause := errors.Unwrap(err)
var pathErr *os.PathError
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &pathErr) || pathErr.Path != path {
t.Fatalf("construction cause = %v, want the original path error without a category branch", cause)
}
}
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) { func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
main := configkit.Input{} main := configkit.Input{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}}) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})

View File

@@ -271,6 +271,8 @@ operation-local plan with no extra resource reads for `$ref` resolution.
## Stage 5 — Bind Effective Contracts In Every Execution Path ## Stage 5 — Bind Effective Contracts In Every Execution Path
Status: complete.
Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and Work in `internal/usecase/runner.go`, prepared execution, engine wiring, and
`smoke_inference_case.go`. `smoke_inference_case.go`.

View File

@@ -144,7 +144,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
endpoint, model := "https://example.invalid/v1", "offline-model" endpoint, model := "https://example.invalid/v1", "offline-model"
schemas := fstest.MapFS{} schemas := fstest.MapFS{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model}, {ID: "offline", Endpoint: &endpoint, Model: &model},
})}}}}, })}}}},
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{ Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
@@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
}}, }},
})}, })},
}}, promptkit.WithSchemaFS(schemas, ".")) }})
if err != nil { if err != nil {
t.Fatalf("construct engine before request resources exist: %v", err) t.Fatalf("construct engine before request resources exist: %v", err)
} }

View File

@@ -13,7 +13,7 @@ import (
type deadlineValidation struct{ deadline time.Time } type deadlineValidation struct{ deadline time.Time }
func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ domain.OutputContract) (validate.PreparedValidation, error) { func (v *deadlineValidation) PrepareValidation(ctx context.Context, _ validate.Input) (validate.PreparedValidation, error) {
v.deadline, _ = ctx.Deadline() v.deadline, _ = ctx.Deadline()
return v, nil return v, nil
} }

View File

@@ -55,19 +55,19 @@ type recordingValidationPreparer struct {
type nilPlanPreparer struct{} type nilPlanPreparer struct{}
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) { func (nilPlanPreparer) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
return nil, nil return nil, nil
} }
func (v *recordingValidationPreparer) PrepareValidation( func (v *recordingValidationPreparer) PrepareValidation(
_ context.Context, _ context.Context,
contract domain.OutputContract, input validate.Input,
) (validate.PreparedValidation, error) { ) (validate.PreparedValidation, error) {
v.prepareCalls++ v.prepareCalls++
if v.prepareErr != nil { if v.prepareErr != nil {
return nil, v.prepareErr return nil, v.prepareErr
} }
v.plan.contract = contract v.plan.contract = input.Contract
return v.plan, nil return v.plan, nil
} }

View File

@@ -49,7 +49,7 @@ func (validator preparedSnapshotBenchmarkValidator) Validate(
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation( func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
context.Context, context.Context,
domain.OutputContract, validate.Input,
) (validate.PreparedValidation, error) { ) (validate.PreparedValidation, error) {
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
} }

View File

@@ -18,6 +18,7 @@ import (
"go.maximumdirect.net/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/capacity"
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog"
"go.maximumdirect.net/promptkit/internal/generation" "go.maximumdirect.net/promptkit/internal/generation"
"go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
@@ -73,6 +74,7 @@ type preparationState struct {
effectiveModel domain.ExecutionTarget effectiveModel domain.ExecutionTarget
targetPresence domain.ExecutionTargetPresence targetPresence domain.ExecutionTargetPresence
effectiveContract domain.OutputContract effectiveContract domain.OutputContract
validationInput validate.Input
start time.Time start time.Time
transientRetries uint64 transientRetries uint64
} }
@@ -353,8 +355,15 @@ func (r *Runner) resolvePreparation(
promptDefinitionHash := promptSelection.hash promptDefinitionHash := promptSelection.hash
effectiveContract, err := resolveOutputContract(ctx, def, req.Validation) effectiveContract, err := resolveOutputContract(ctx, def, req.Validation)
if err != nil { if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
return nil, contextErr
}
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
} }
validationInput, err := bindValidationInput(def, effectiveContract, req.Validation != nil)
if err != nil {
return nil, fmt.Errorf("%w: prompt %q: %v", ErrInvalidRequest, def.ID, err)
}
selectedProfileID := req.ProfileID selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" { if strings.TrimSpace(selectedProfileID) == "" {
@@ -394,6 +403,7 @@ func (r *Runner) resolvePreparation(
effectiveModel: effectiveModel, effectiveModel: effectiveModel,
targetPresence: targetPresence, targetPresence: targetPresence,
effectiveContract: effectiveContract, effectiveContract: effectiveContract,
validationInput: validationInput,
start: start, start: start,
transientRetries: transientRetries, transientRetries: transientRetries,
}, nil }, nil
@@ -404,7 +414,7 @@ func (r *Runner) completePreparation(
req domain.RunRequest, req domain.RunRequest,
state *preparationState, state *preparationState,
) (*preparedOperation, error) { ) (*preparedOperation, error) {
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract) validationPlan, err := r.prepareValidation(ctx, state.definition.ID, state.validationInput)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -425,14 +435,15 @@ func (r *Runner) completePreparation(
func (r *Runner) prepareValidation( func (r *Runner) prepareValidation(
ctx context.Context, ctx context.Context,
contract domain.OutputContract, promptID string,
input validate.Input,
) (validate.PreparedValidation, error) { ) (validate.PreparedValidation, error) {
if r.validator == nil { if r.validator == nil {
return noOpPreparedValidation{contract: contract}, nil return noOpPreparedValidation{contract: input.Contract}, nil
} }
plan, err := r.validator.PrepareValidation(ctx, contract) plan, err := r.validator.PrepareValidation(ctx, input)
if err != nil { if err != nil {
return nil, &OperationError{Category: ErrValidation, Cause: err} return nil, &OperationError{Category: ErrValidation, Cause: fmt.Errorf("prompt %q: %w", promptID, err)}
} }
if plan == nil { if plan == nil {
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation) return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
@@ -440,6 +451,31 @@ func (r *Runner) prepareValidation(
return plan, nil return plan, nil
} }
func bindValidationInput(def *domain.PromptDefinition, contract domain.OutputContract, replaced bool) (validate.Input, error) {
input := validate.Input{Contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return input, nil
}
if contract.SchemaFile != nil && !replaced {
if def.SchemaFile == nil {
return validate.Input{}, fmt.Errorf("schema_file %q has no resource binding", *contract.SchemaFile)
}
input.File = def.SchemaFile
return input, nil
}
if contract.SchemaPath != "" {
if def.SchemaRoot == nil {
return validate.Input{}, errors.New("schema_path requires an applicable schema_root")
}
resolved, _, err := filecatalog.ResolveFSPath(def.SchemaRoot.Path, def.SchemaRoot.Path, contract.SchemaPath)
if err != nil {
return validate.Input{}, fmt.Errorf("invalid schema_path %q: %w", contract.SchemaPath, err)
}
input.File = &domain.ResourceBinding{FS: def.SchemaRoot.FS, Path: resolved}
}
return input, nil
}
func (r *Runner) structuredOutputFromValidationPlan( func (r *Runner) structuredOutputFromValidationPlan(
def *domain.PromptDefinition, def *domain.PromptDefinition,
contract domain.OutputContract, contract domain.OutputContract,

View File

@@ -13,6 +13,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"testing/fstest"
"text/template" "text/template"
"time" "time"
@@ -246,7 +247,8 @@ type fakeValidator struct {
validateCalls int validateCalls int
} }
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) { func (f *fakeValidator) PrepareValidation(_ context.Context, input validate.Input) (validate.PreparedValidation, error) {
contract := input.Contract
var schemaDocument any var schemaDocument any
if contract.ValidationMode == domain.ValidationJSONSchema { if contract.ValidationMode == domain.ValidationJSONSchema {
f.schemaLoads++ f.schemaLoads++
@@ -2519,7 +2521,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
staticArtifactReader{}, staticArtifactReader{},
staticRenderer{}, staticRenderer{},
scheduledClient, scheduledClient,
validate.NewStandardValidator("."), validate.NewValidator(),
NewDefaultOutputRepairer(scheduledClient), NewDefaultOutputRepairer(scheduledClient),
admitter, admitter,
) )
@@ -2614,6 +2616,7 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
DefaultProfile: "exec", DefaultProfile: "exec",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "."},
Validation: domain.OutputContract{ Validation: domain.OutputContract{
ValidationMode: mode, ValidationMode: mode,
RepairAttempts: attempts, RepairAttempts: attempts,

View File

@@ -0,0 +1,51 @@
package usecase
import (
"context"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
fileName := "configured.json"
def := &domain.PromptDefinition{
ID: "collection/prompt",
SchemaRoot: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"},
SchemaFile: &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "origin/configured.json"},
Validation: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaFile: &fileName},
}
configured, err := bindValidationInput(def, def.Validation, false)
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
t.Fatalf("configured binding = %+v, %v", configured, err)
}
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
contract, err := resolveOutputContract(context.Background(), def, inline)
if err != nil {
t.Fatal(err)
}
selected, err := bindValidationInput(def, contract, true)
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
t.Fatalf("inline replacement = %+v, %v", selected, err)
}
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
contract, err = resolveOutputContract(context.Background(), def, path)
if err != nil {
t.Fatal(err)
}
selected, err = bindValidationInput(def, contract, true)
if err != nil || selected.File == nil || selected.File.Path != "collection/schemas/runtime.json" || selected.Contract.SchemaFile != nil {
t.Fatalf("path replacement = %+v, %v", selected, err)
}
def.SchemaRoot = nil
if _, err := bindValidationInput(def, contract, true); err == nil || !strings.Contains(err.Error(), "schema_root") {
t.Fatalf("missing root = %v", err)
}
def.SchemaRoot = &domain.ResourceBinding{FS: fstest.MapFS{}, Path: "collection/schemas"}
contract.SchemaPath = "../outside.json"
if _, err := bindValidationInput(def, contract, true); err == nil {
t.Fatal("escaping path accepted")
}
}

View File

@@ -51,18 +51,17 @@ func prepareSmokeInferenceCase(
), ),
}, nil }, nil
case domain.SmokeTestStructuredOutput: case domain.SmokeTestStructuredOutput:
validator := validate.NewFSValidator(smokeSchemaFS, ".") validator := validate.NewValidator()
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{ jsonPlan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
}) }})
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err) return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
} }
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{ plan, err := validator.PrepareValidation(ctx, validate.Input{Contract: domain.OutputContract{
Format: domain.FormatJSON, Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema, ValidationMode: domain.ValidationJSONSchema,
SchemaPath: smokeStructuredSchemaPath, }, File: &domain.ResourceBinding{FS: smokeSchemaFS, Path: smokeStructuredSchemaPath}})
})
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err) return nil, fmt.Errorf("%w: prepare private structured-output schema: %w", ErrValidation, err)
} }

View File

@@ -42,7 +42,7 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
type selectionTestValidator struct{ state *selectionTestCollaborators } type selectionTestValidator struct{ state *selectionTestCollaborators }
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) { func (v selectionTestValidator) PrepareValidation(context.Context, validate.Input) (validate.PreparedValidation, error) {
v.state.validationCalls++ v.state.validationCalls++
return nil, errors.New("validation must not occur during target selection") return nil, errors.New("validation must not occur during target selection")
} }
@@ -359,13 +359,13 @@ func smokeSelectionConfig() map[string]any {
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}}, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
"session_id": "{{.vars.session}}", "session_id": "{{.vars.session}}",
"output": map[string]any{ "output": map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", "format": "json", "validation_mode": "json_schema", "schema": false,
}, },
}, },
map[string]any{"id": "backend.prompt", map[string]any{"id": "backend.prompt",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{ "output": map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", "format": "json", "validation_mode": "json_schema", "schema": false,
}, },
}, },
}, },

View File

@@ -3,7 +3,6 @@ package promptkit_test
import ( import (
"context" "context"
"testing" "testing"
"testing/fstest"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
@@ -52,8 +51,7 @@ func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) {
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"}) session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true) cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"}) cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}} schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object"}}, false)
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, "."))
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"}) schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
for name, prepared := range map[string]*promptkit.PreparedRun{ for name, prepared := range map[string]*promptkit.PreparedRun{
"repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}), "repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}),

View File

@@ -242,7 +242,7 @@ func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
func TestRunObservationValidationPreparationError(t *testing.T) { func TestRunObservationValidationPreparationError(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}) engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
var events []promptkit.ExecutionEvent var events []promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: map[string]any{"type": "invalid"}}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 { if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
t.Fatalf("result=%+v error=%v events=%+v", result, err, events) t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
} }

View File

@@ -16,7 +16,7 @@ import (
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine { func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
t.Helper() t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{
Source: configkit.Mapping("requirements contract", map[string]any{ Source: configkit.Mapping("requirements contract", map[string]any{
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles), "prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
}), }),

View File

@@ -5,6 +5,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"testing/fstest"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
@@ -12,7 +13,7 @@ import (
func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) { func TestResourceInspectionIsolationAndNullInheritance(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{}, ".")}}}, Main: &configkit.Input{Source: configkit.Mapping("resources", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}}, "backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.test/v1", "api_key_env": "UNSET_INSPECTION_KEY", "extra_params": map[string]any{"nested": map[string]any{"array": []any{"captured"}}}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}}, "profiles": []any{map[string]any{"id": "parent", "backend": "local", "endpoint": "https://parent.test/v1", "extra_params": map[string]any{"parent": true}, "model": "model"}, map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil}, map[string]any{"id": "partial"}},
"prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, "prompts": []any{map[string]any{"id": "reference", "default_profile": "missing", "messages": []any{map[string]any{"role": "user", "content": "{{.missing}}"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "missing.json"}}, map[string]any{"id": "ready", "inputs": []any{map[string]any{"name": "x", "required": false}}, "messages": []any{map[string]any{"role": "user", "content": "hello", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},

View File

@@ -0,0 +1,74 @@
package promptkit_test
import (
"context"
"errors"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestRuntimeSchemaReplacementUsesPromptScope(t *testing.T) {
app := fstest.MapFS{"schema.json": {Data: []byte(`{"type":"string"}`)}}
collection := fstest.MapFS{
"collection/placeholder": {Data: []byte("context")},
"collection/root/schema.json": {Data: []byte(`{"type":"integer"}`)},
}
filesystem := configkit.NewFilesystem("collection", collection)
input := configkit.Input{Source: configkit.Mapping("prompts", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"collections": map[string]any{"team": map[string]any{"schema_root": "root", "prompts": []any{map[string]any{
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"},
}}}},
}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "collection"}}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `42`}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(app, ".")}}}, Main: &input}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
base := promptkit.RunRequest{PromptID: "team/prompt", ProfileID: "profile"}
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrValidation) {
t.Fatalf("configured missing file = %v", err)
}
if len(client.requests) != 0 {
t.Fatal("preparation called model")
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, Schema: true}
result, err := engine.Run(context.Background(), base)
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("inline replacement = %+v, %v", result, err)
}
if got := client.requests[0].StructuredOutput.JSONSchema.Schema; got != true {
t.Fatalf("inline submitted schema = %#v", got)
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}
if err := engine.ValidateRequest(context.Background(), base); err != nil {
t.Fatalf("runtime path validation = %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), base)
if err != nil {
t.Fatal(err)
}
result, err = engine.RunPrepared(context.Background(), prepared)
if err != nil || result.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("root replacement = %+v, %v", result, err)
}
if got := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["type"]; got != "integer" {
t.Fatalf("selected collection root schema = %#v", got)
}
base.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema}
if _, err := engine.Prepare(context.Background(), base); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("missing replacement selection = %v", err)
}
}
func TestRuntimeSchemaPathWithoutRootIsInvalidRequest(t *testing.T) {
engine := newSimpleEngine(t, "hello")
request := promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "schema.json"}}
if _, err := engine.Prepare(context.Background(), request); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("missing root = %v", err)
}
}

View File

@@ -24,18 +24,17 @@ func (rejectWireSchemaLoad) Load(uri string) (any, error) {
} }
func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) { func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
firstDir, secondDir := t.TempDir(), t.TempDir() secondDir := t.TempDir()
first, second := filepath.Join(firstDir, "schema.json"), filepath.Join(secondDir, "schema.json") second := filepath.Join(secondDir, "schema.json")
if err := os.WriteFile(first, []byte(`{"type":"string"}`), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil { if err := os.WriteFile(second, []byte(`{"$defs":{"value":{"type":"integer"}},"$ref":"#/$defs/value"}`), 0600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
})} })}
filesystem := configkit.NewFilesystem("schema", os.DirFS(secondDir))
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
var payload map[string]any var payload map[string]any
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
@@ -43,7 +42,7 @@ func TestSchemaFilePreparationAndNativeSubmission(t *testing.T) {
} }
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"3"}}]}`))}, nil
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(first), promptkit.WithSchemaFile(second)) engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -91,9 +90,11 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
} }
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(path)) filesystem := configkit.NewFilesystem("schema", os.DirFS(filepath.Dir(path)))
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -115,22 +116,19 @@ func TestSchemaFilePreparationDetachesLargeAnnotation(t *testing.T) {
func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) { func TestSchemaFileRejectsInvalidSourcesAndExternalReferences(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
for _, name := range []string{filepath.Join(dir, "missing.json"), dir} {
if _, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithSchemaFile(name)); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("source %q = %v", name, err)
}
}
root := filepath.Join(dir, "schema.json") root := filepath.Join(dir, "schema.json")
if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil { if err := os.WriteFile(root, []byte(`{"$defs":{"unused":{"$ref":"other.json"}}}`), 0600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{ input := configkit.Input{Source: configkit.Mapping("schema test", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}}},
})} })}
filesystem := configkit.NewFilesystem("schema", os.DirFS(dir))
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
calls := 0 calls := 0
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })} client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })}
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithSchemaFile(root)) engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -12,6 +12,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"testing/fstest"
"time" "time"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
@@ -308,7 +309,10 @@ type failingSchemaFS struct {
calls int calls int
} }
func (source *failingSchemaFS) Open(string) (fs.File, error) { func (source *failingSchemaFS) Open(name string) (fs.File, error) {
if name == "." {
return (fstest.MapFS{}).Open(name)
}
source.mu.Lock() source.mu.Lock()
source.calls++ source.calls++
source.mu.Unlock() source.mu.Unlock()
@@ -346,8 +350,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
}, },
}}, }},
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemaSource, "schemas")}}}}},
promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader), promptkit.WithSchemaFS(schemaSource, "schemas")) promptkit.WithLLMClient(client), promptkit.WithArtifactReader(reader))
if err != nil { if err != nil {
t.Fatalf("construct engine: %v", err) t.Fatalf("construct engine: %v", err)
} }

View File

@@ -6,7 +6,6 @@ import (
"path/filepath" "path/filepath"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/fstest"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
@@ -146,7 +145,6 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
"schema": onePrompt("schema", schemaPrompt)["schema"], "schema": onePrompt("schema", schemaPrompt)["schema"],
} }
engine := requirementsEngine(t, prompts, profiles, engine := requirementsEngine(t, prompts, profiles,
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
) )