From 38a2dcca525676895c69b0ffc730830bbad50c57 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 21 Nov 2026 14:12:34 +0000 Subject: [PATCH] Separate output contract inputs from resolved values --- convert.go | 11 +-- docs/roadmap/implementation.md | 3 + internal/domain/domain.go | 11 ++- internal/usecase/output_contract_test.go | 85 +++++++++++++++++++----- internal/usecase/runner.go | 4 +- internal/usecase/schema_binding_test.go | 4 +- types.go | 16 ++++- 7 files changed, 103 insertions(+), 31 deletions(-) diff --git a/convert.go b/convert.go index cc66548..3050905 100644 --- a/convert.go +++ b/convert.go @@ -318,21 +318,16 @@ func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) } } -func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract { +func toDomainOutputContractPtr(contract *OutputContractInput) *domain.OutputContractInput { if contract == nil { return nil } - out := toDomainOutputContract(*contract) - return &out -} - -func toDomainOutputContract(contract OutputContract) domain.OutputContract { - return domain.OutputContract{ + return &domain.OutputContractInput{ Format: domain.OutputFormat(contract.Format), ValidationMode: domain.ValidationMode(contract.ValidationMode), Schema: contract.Schema, SchemaPath: contract.SchemaPath, - RepairAttempts: contract.RepairAttempts, + RepairAttempts: copyIntPtr(contract.RepairAttempts), } } diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2217b46..e5ffdc9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -261,6 +261,9 @@ with no change to actual-attempt counters or the repair loop. ## Stage 3 — Separate Runtime Contract Inputs From Resolved Contracts +Status: complete. Internal use-case tests and the facade build passed; existing +public test and example call sites remain scheduled for migration below. + Focus: `types.go`, `convert.go`, `internal/domain/domain.go`, request ownership, `internal/usecase/runner.go`, and focused runtime tests. diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 7c4aeaf..3fc7dcd 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -74,7 +74,7 @@ type RunRequest struct { Inputs map[string]ArtifactRef Vars map[string]string Execution *ExecutionTargetOverride - Validation *OutputContract + Validation *OutputContractInput AppendedMessages []RenderedMessage TransientRetries *int Observer ExecutionObserver `json:"-"` @@ -261,6 +261,15 @@ type OutputContract struct { RepairAttempts int `yaml:"repair_attempts"` } +// OutputContractInput is a request replacement before repair defaults resolve. +type OutputContractInput struct { + Format OutputFormat + ValidationMode ValidationMode + Schema any + SchemaPath string + RepairAttempts *int +} + // ResourceBinding identifies a resource in its originating filesystem. type ResourceBinding struct { FS fs.FS diff --git a/internal/usecase/output_contract_test.go b/internal/usecase/output_contract_test.go index 8bd3644..4dfc259 100644 --- a/internal/usecase/output_contract_test.go +++ b/internal/usecase/output_contract_test.go @@ -6,6 +6,7 @@ import ( "testing" "go.maximumdirect.net/promptkit/internal/domain" + "go.maximumdirect.net/promptkit/internal/validate" ) type outputContractTestCollaborators struct { @@ -38,7 +39,7 @@ func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) { func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) { value := map[string]any{"properties": map[string]any{"name": map[string]any{"type": "string"}}} - override := domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value} + override := domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value} contract, err := resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override) if err != nil { t.Fatal(err) @@ -59,26 +60,26 @@ func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) { func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) { tests := []struct { name string - override domain.OutputContract + override domain.OutputContractInput want domain.OutputContract }{ { name: "empty replacement format defaults to text", - override: domain.OutputContract{ValidationMode: domain.ValidationNone}, + override: domain.OutputContractInput{ValidationMode: domain.ValidationNone}, want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone}, }, { name: "markdown basic replacement", - override: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic}, - want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic}, + override: domain.OutputContractInput{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic}, + want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic, RepairAttempts: 1}, }, { name: "json replacement preserves non-schema fields", - override: domain.OutputContract{ + override: domain.OutputContractInput{ Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, SchemaPath: "ignored.json", - RepairAttempts: 2, + RepairAttempts: repairBudget(2), }, want: domain.OutputContract{ Format: domain.FormatJSON, @@ -121,14 +122,14 @@ func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) { func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) { tests := []struct { name string - override domain.OutputContract + override domain.OutputContractInput }{ - {name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}}, - {name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}}, - {name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}}, - {name: "repair attempts above maximum", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 4}}, - {name: "none validation with repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: 1}}, - {name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}}, + {name: "unsupported format", override: domain.OutputContractInput{Format: "binary", ValidationMode: domain.ValidationNone}}, + {name: "empty validation mode", override: domain.OutputContractInput{Format: domain.FormatText}}, + {name: "negative repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(-1)}}, + {name: "repair attempts above maximum", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(4)}}, + {name: "none validation with repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(1)}}, + {name: "json schema without path", override: domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}}, } for _, tt := range tests { @@ -175,7 +176,7 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) { PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), - Validation: &domain.OutputContract{ + Validation: &domain.OutputContractInput{ Format: domain.OutputFormat("binary"), ValidationMode: domain.ValidationNone, }, @@ -189,6 +190,60 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) { assertOutputContractCompletionSkipped(t, collaborators) } +func repairBudget(value int) *int { return &value } + +func TestReplacementRepairExecution(t *testing.T) { + for _, tc := range []struct { + name string + budget *int + wantCalls int + wantStatus domain.ValidationStatus + }{ + {"default", nil, 2, domain.ValidationPassed}, + {"explicit zero", repairBudget(0), 1, domain.ValidationFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + client := &sequenceLLM{responses: []*domain.GenerateResponse{{Content: "not json"}, {Content: `{"ok":true}`}}} + runner := NewRunnerWithRepairer(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), defaultRenderer(), client, validate.NewValidator(), NewDefaultOutputRepairer(client), nil) + input := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, RepairAttempts: tc.budget} + prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Validation: input}) + if err != nil { + t.Fatal(err) + } + input.ValidationMode = domain.ValidationNone + input.RepairAttempts = repairBudget(3) + result, err := runner.RunPrepared(context.Background(), prepared) + if err != nil || result.Validation.Status != tc.wantStatus || len(client.requests) != tc.wantCalls || result.Validation.RepairAttempts != tc.wantCalls-1 { + t.Fatalf("result = %+v, calls = %d, error = %v", result, len(client.requests), err) + } + }) + } +} + +func TestResolveReplacementRepairBudget(t *testing.T) { + configured := &domain.PromptDefinition{Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 3}} + for _, tc := range []struct { + name string + input domain.OutputContractInput + want int + }{ + {"none", domain.OutputContractInput{ValidationMode: domain.ValidationNone}, 0}, + {"basic", domain.OutputContractInput{ValidationMode: domain.ValidationBasic}, 1}, + {"json", domain.OutputContractInput{ValidationMode: domain.ValidationJSON}, 1}, + {"schema", domain.OutputContractInput{ValidationMode: domain.ValidationJSONSchema, Schema: true}, 1}, + {"explicit zero", domain.OutputContractInput{ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(0)}, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveOutputContract(context.Background(), configured, &tc.input) + if err != nil || got.RepairAttempts != tc.want { + t.Fatalf("contract = %+v, %v", got, err) + } + }) + } +} + func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) { t.Helper() if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 || diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 69e727a..1a675fc 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -685,10 +685,10 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error return nil } -func resolveOutputContract(ctx context.Context, def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) { +func resolveOutputContract(ctx context.Context, def *domain.PromptDefinition, override *domain.OutputContractInput) (domain.OutputContract, error) { contract := def.Validation if override != nil { - contract = *override + contract = domain.OutputContract{Format: override.Format, ValidationMode: override.ValidationMode, Schema: override.Schema, SchemaPath: override.SchemaPath, RepairAttempts: domain.ResolveRepairAttempts(override.ValidationMode, override.RepairAttempts)} if contract.Format == "" { contract.Format = domain.FormatText } diff --git a/internal/usecase/schema_binding_test.go b/internal/usecase/schema_binding_test.go index 0795497..233dff3 100644 --- a/internal/usecase/schema_binding_test.go +++ b/internal/usecase/schema_binding_test.go @@ -21,7 +21,7 @@ func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) { if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" { t.Fatalf("configured binding = %+v, %v", configured, err) } - inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false} + inline := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false} contract, err := resolveOutputContract(context.Background(), def, inline) if err != nil { t.Fatal(err) @@ -30,7 +30,7 @@ func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) { if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil { t.Fatalf("inline replacement = %+v, %v", selected, err) } - path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"} + path := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"} contract, err = resolveOutputContract(context.Background(), def, path) if err != nil { t.Fatal(err) diff --git a/types.go b/types.go index 5e173c3..14350fb 100644 --- a/types.go +++ b/types.go @@ -160,7 +160,7 @@ type RunRequest struct { Execution *ExecutionTargetOverride // Validation optionally replaces the prompt's complete output contract. It // does not merge individual fields. Nil uses the prompt contract. - Validation *OutputContract + Validation *OutputContractInput // AppendedMessages are already-rendered messages appended in caller order // after every prompt definition message. Promptkit neither templates nor // resolves files in them. Content must be valid UTF-8 and is preserved @@ -490,8 +490,7 @@ type ExecutionTargetPresence struct { TimeoutSeconds bool `json:"timeout_seconds"` } -// OutputContract defines output and validation requirements and has a stable -// JSON representation. +// OutputContract describes resolved output and validation requirements. // // A non-nil RunRequest.Validation replaces the complete prompt contract. It // does not merge fields. The public Engine performs bounded correction after a @@ -520,6 +519,17 @@ type OutputContract struct { RepairAttempts int `json:"repair_attempts"` } +// OutputContractInput replaces a prompt's complete output contract. An omitted +// repair budget defaults to one for active validation and zero for none; +// an explicit zero disables repair. Schema selection follows OutputContract. +type OutputContractInput struct { + Format OutputFormat `json:"format"` + ValidationMode ValidationMode `json:"validation_mode"` + Schema any `json:"schema,omitempty"` + SchemaPath string `json:"schema_path"` + RepairAttempts *int `json:"repair_attempts,omitempty"` +} + // ValidationResult represents a completed output check and has a stable JSON // representation. An operational inability to perform validation is returned // as ErrValidation instead of a ValidationResult.