diff --git a/convert.go b/convert.go index 8fd5017..cc66548 100644 --- a/convert.go +++ b/convert.go @@ -283,7 +283,7 @@ func fromBackendResource(v *configuration.BackendResource) *BackendInspection { } func fromPromptResource(v *configuration.PromptResource) *PromptInspection { - out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}} + out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: v.Output.RepairAttempts}} if v.Inputs != nil { inputs := make([]PromptInputInspection, len(*v.Inputs)) for i, input := range *v.Inputs { diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 4615938..2217b46 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -233,6 +233,8 @@ validation, not an approval gate before Stage 2. ## Stage 2 — Resolve Configuration Repair Defaults +Status: complete. Focused configuration, domain, and public source tests passed. + Focus: `internal/domain`, configuration output decoding/resolution/capture, resource inspection, and their narrow tests. diff --git a/internal/configuration/load.go b/internal/configuration/load.go index 7bc40ab..6cf96fa 100644 --- a/internal/configuration/load.go +++ b/internal/configuration/load.go @@ -108,7 +108,7 @@ type promptOutputConfig struct { Schema any `yaml:"schema"` SchemaFile *string `yaml:"schema_file"` SchemaPath *string `yaml:"schema_path"` - RepairAttempts int `yaml:"repair_attempts"` + RepairAttempts *int `yaml:"repair_attempts"` } // fieldState distinguishes omission, an explicit value, and an explicit null. @@ -124,7 +124,6 @@ const ( type loaded struct { result *configkit.Result[document] promptMetadata map[string][]promptInputPresence - promptRepair map[string]bool prompts map[string]*promptEntry registry *backend.Registry backendRecords map[string]backendConfig @@ -239,7 +238,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] { Sensitive: secretProviderPath, Resolve: func(config *document, resolution *configkit.Resolution) error { for _, resources := range resourceSets(config) { - capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair) + capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata) } for _, resources := range resourceSets(config) { resolveBackends(resources.backends, resources.bases.backends, resolution) @@ -258,7 +257,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] { func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { state := &loaded{ promptMetadata: make(map[string][]promptInputPresence), - promptRepair: make(map[string]bool), } result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{ Underlays: underlays, @@ -328,7 +326,7 @@ type promptInputPresence struct { description fieldState } -func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) { +func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) { for promptIndex, prompt := range prompts { inputs := prompt.Inputs states := make([]promptInputPresence, len(inputs)) @@ -342,7 +340,6 @@ func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, name } } destination[indexedResourceID(namespace, prompt.ID)] = states - repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue } } diff --git a/internal/configuration/prompts.go b/internal/configuration/prompts.go index 7156a70..c788615 100644 --- a/internal/configuration/prompts.go +++ b/internal/configuration/prompts.go @@ -247,7 +247,7 @@ func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domai contract := domain.OutputContract{ Format: domain.OutputFormat(*output.Format), ValidationMode: domain.ValidationMode(*output.ValidationMode), - RepairAttempts: output.RepairAttempts, + RepairAttempts: domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), output.RepairAttempts), Schema: jsonvalue.CloneTrusted(output.Schema), SchemaFile: stringPointerValue(output.SchemaFile), } diff --git a/internal/configuration/prompts_test.go b/internal/configuration/prompts_test.go index 9c23f3c..507ce15 100644 --- a/internal/configuration/prompts_test.go +++ b/internal/configuration/prompts_test.go @@ -3,6 +3,7 @@ package configuration import ( "context" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -117,11 +118,96 @@ func TestInspectPromptResourcePresence(t *testing.T) { if err != nil { t.Fatal(err) } - if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil { + if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil { t.Fatalf("prompt resource = %+v", got) } } +func TestConfiguredRepairBudgets(t *testing.T) { + for _, tc := range []struct { + name, mode string + budget any + include bool + want int + invalid bool + }{ + {name: "none omitted", mode: "none", want: 0}, + {name: "basic omitted", mode: "basic", want: 1}, + {name: "json omitted", mode: "json", want: 1}, + {name: "schema omitted", mode: "json_schema", want: 1}, + {name: "null", mode: "json", include: true, budget: nil, want: 1}, + {name: "zero", mode: "basic", include: true, budget: 0, want: 0}, + {name: "two", mode: "basic", include: true, budget: 2, want: 2}, + {name: "three", mode: "basic", include: true, budget: 3, want: 3}, + {name: "negative", mode: "basic", include: true, budget: -1, invalid: true}, + {name: "excess", mode: "basic", include: true, budget: 4, invalid: true}, + {name: "none positive", mode: "none", include: true, budget: 1, invalid: true}, + {name: "unknown mode", mode: "unknown", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + output := map[string]any{"format": "text", "validation_mode": tc.mode} + if tc.mode == "json_schema" { + output["schema"] = true + } + if tc.include { + output["repair_attempts"] = tc.budget + } + state, err := load(mapping("repair", map[string]any{"prompts": []any{map[string]any{ + "id": "test", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output, + }}}), nil) + if tc.invalid { + if err == nil { + t.Fatal("expected invalid output contract") + } + return + } + if err != nil { + t.Fatal(err) + } + definition, _, err := state.prompt("test") + if err != nil || definition.Validation.RepairAttempts != tc.want { + t.Fatalf("captured budget: %+v, %v", definition, err) + } + report, err := state.InspectPrompt(context.Background(), "test") + if err != nil || report.Output.RepairAttempts != tc.want { + t.Fatalf("inspection: %+v, %v", report, err) + } + if got := state.result.Config.Prompts[0].Output.RepairAttempts; got == nil || *got != tc.want { + t.Fatalf("effective budget = %v", got) + } + var effective strings.Builder + if err := state.result.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), fmt.Sprintf("repair_attempts: %d", tc.want)) { + t.Fatalf("effective inspection = %q, %v", effective.String(), err) + } + }) + } +} + +func TestNullClearsInheritedRepairBudget(t *testing.T) { + state, err := load(mapping("inherited repair", map[string]any{"prompts": []any{ + map[string]any{"id": "parent", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 3}}, + map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "output": map[string]any{"repair_attempts": nil}}, + }}), nil) + if err != nil { + t.Fatal(err) + } + child, _, err := state.prompt("child") + if err != nil || child.Validation.RepairAttempts != 1 { + t.Fatalf("child = %+v, %v", child, err) + } +} + +func TestYAMLRepairBudgetNull(t *testing.T) { + state, err := load(configkit.Input{Source: configkit.Bytes("prompt", []byte("prompts:\n - id: example\n messages:\n - role: user\n content: hello\n output:\n format: text\n validation_mode: basic\n repair_attempts: null\n"))}, nil) + if err != nil { + t.Fatal(err) + } + definition, _, err := state.prompt("example") + if err != nil || definition.Validation.RepairAttempts != 1 { + t.Fatalf("prompt = %+v, %v", definition, err) + } +} + func TestLoadHeaderlessImportedItem(t *testing.T) { files := fstest.MapFS{ "main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")}, diff --git a/internal/configuration/resolve_prompts.go b/internal/configuration/resolve_prompts.go index 3928276..4fa9424 100644 --- a/internal/configuration/resolve_prompts.go +++ b/internal/configuration/resolve_prompts.go @@ -40,6 +40,10 @@ func resolvePrompts(entries []promptConfig, base configkit.Path, resolution *con if entry.Output != nil && entry.Output.SchemaPath != nil { recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath)) } + if output := entry.Output; output != nil && output.ValidationMode != nil && output.RepairAttempts == nil { + attempts := domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), nil) + configkit.Normalize(resolution, path.Key("output").Key("repair_attempts"), &output.RepairAttempts, &attempts, "Resolve repair budget") + } for j := range entry.Messages { message := &entry.Messages[j] messagePath := path.Key("messages").Index(j) diff --git a/internal/configuration/resources.go b/internal/configuration/resources.go index 728577e..c1dd0cf 100644 --- a/internal/configuration/resources.go +++ b/internal/configuration/resources.go @@ -63,7 +63,7 @@ type PromptOutputResource struct { Schema any SchemaFile *string SchemaPath *string - RepairAttempts *int + RepairAttempts int } func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) { @@ -95,11 +95,7 @@ func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResou result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL} } } - result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, Schema: entry.definition.Validation.Schema, SchemaFile: raw.Output.SchemaFile, SchemaPath: raw.Output.SchemaPath} - if state.promptRepair[id] { - value := raw.Output.RepairAttempts - result.Output.RepairAttempts = &value - } + result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, Schema: entry.definition.Validation.Schema, SchemaFile: raw.Output.SchemaFile, SchemaPath: raw.Output.SchemaPath, RepairAttempts: entry.definition.Validation.RepairAttempts} return result, nil } diff --git a/internal/domain/output_contract.go b/internal/domain/output_contract.go index 05bf33c..078bb54 100644 --- a/internal/domain/output_contract.go +++ b/internal/domain/output_contract.go @@ -8,6 +8,19 @@ import ( const maxOutputRepairAttempts = 3 +// ResolveRepairAttempts selects a concrete budget for a validation mode. +func ResolveRepairAttempts(mode ValidationMode, attempts *int) int { + if attempts != nil { + return *attempts + } + switch mode { + case ValidationBasic, ValidationJSON, ValidationJSONSchema: + return 1 + default: + return 0 + } +} + // ValidateOutputContract validates source-neutral output-contract invariants. func ValidateOutputContract(contract OutputContract) error { switch contract.Format { diff --git a/prompt_sources_contract_test.go b/prompt_sources_contract_test.go index a917561..040830c 100644 --- a/prompt_sources_contract_test.go +++ b/prompt_sources_contract_test.go @@ -46,7 +46,7 @@ func TestTypedSourcesComposeAndCaptureValues(t *testing.T) { if err != nil { t.Fatal(err) } - if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID != "local" || prompt.Description == nil || *prompt.Description != "description" || prompt.Inputs == nil || len(*prompt.Inputs) != 1 || (*prompt.Inputs)[0].Name != "input" || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required || (*prompt.Inputs)[0].Description == nil || *(*prompt.Inputs)[0].Description != "" || len(prompt.Messages) != 2 || prompt.Messages[0].Content != "hello {{.name}}" || prompt.Messages[1].Content != "second" || prompt.Messages[0].CacheControl.TTL == nil || *prompt.Messages[0].CacheControl.TTL != "" || prompt.OutputContract.RepairAttempts == nil || *prompt.OutputContract.RepairAttempts != 0 { + if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID != "local" || prompt.Description == nil || *prompt.Description != "description" || prompt.Inputs == nil || len(*prompt.Inputs) != 1 || (*prompt.Inputs)[0].Name != "input" || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required || (*prompt.Inputs)[0].Description == nil || *(*prompt.Inputs)[0].Description != "" || len(prompt.Messages) != 2 || prompt.Messages[0].Content != "hello {{.name}}" || prompt.Messages[1].Content != "second" || prompt.Messages[0].CacheControl.TTL == nil || *prompt.Messages[0].CacheControl.TTL != "" || prompt.OutputContract.RepairAttempts != 0 { t.Fatalf("prompt = %+v", prompt) } prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "child", Vars: map[string]string{"name": "reader"}}) diff --git a/resource_inspection.go b/resource_inspection.go index 9b14ca8..609ed02 100644 --- a/resource_inspection.go +++ b/resource_inspection.go @@ -83,5 +83,5 @@ type PromptOutputInspection struct { Schema any `json:"schema,omitempty" yaml:"schema,omitempty"` SchemaFile *string `json:"schema_file,omitempty" yaml:"schema_file,omitempty"` SchemaPath *string `json:"schema_path,omitempty" yaml:"schema_path,omitempty"` - RepairAttempts *int `json:"repair_attempts,omitempty" yaml:"repair_attempts,omitempty"` + RepairAttempts int `json:"repair_attempts" yaml:"repair_attempts"` }