diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index 89e0f70..49796ac 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -7,7 +7,7 @@ is owned by the [CLI reference](../cli.md) and [operations guide](../operations. ## Single-Report Flow -`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. Output preflight validates the final filename, permits only an absent or regular final destination, and validates the bounded same-directory temporary form without creating a missing parent. It then validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. Profile inspection requires a model, permits an empty backend identity for endpoint-only profiles, and leaves inherited resolution and optional credential sources to Promptkit. The resolved profile, backend, and model are carried in the active result. +`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. Output preflight validates the final filename, permits only an absent or regular final destination, and validates the bounded same-directory temporary form without creating a missing parent. It then validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. Profile inspection requires a model, permits an empty backend identity for endpoint-only profiles, and leaves inherited resolution and optional credential sources to Promptkit. The resolved profile, backend, model, and actual repair count (once a completed execution exists) are carried in the active result; the configured repair budget remains part of the exact prompt contract. The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit only against the inspected prompt and profile, reconciles the preparation callback and completed result with that identity and the prepared report schema, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` writes the completed Markdown through a same-directory temporary file, rechecks the final destination and context after close and immediately before the atomic rename. Only after that write succeeds does single-report notification run. diff --git a/docs/internal/prepared-report.md b/docs/internal/prepared-report.md index 5db286a..2ec9426 100644 --- a/docs/internal/prepared-report.md +++ b/docs/internal/prepared-report.md @@ -21,8 +21,9 @@ Before accepting generated JSON, the execution boundary reconciles the prepared report definition, inspected prompt hash and selected profile identity, the one preparation callback, and the completed Promptkit result. The callback and completion must agree on prompt, profile, backend, model, and rendered/input -hashes; the callback output and completed validation must name the prepared -report's JSON Schema. A mismatch produces no rendered Markdown and leaves +hashes; the callback output must also carry the prepared report's configured +repair budget, while the completed validation records the actual corrective +calls used within that budget. A mismatch produces no rendered Markdown and leaves results with only the inspected safe identity. Single-report generation executes one prepared profile and publishes its diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0d6f86a..173daf1 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -286,6 +286,8 @@ GOWORK=off go test -race -count=1 ./internal/adapters/promptkit ### Stage 5: Carry Repair Provenance Through Application Workflows +Status: Complete. + Purpose: make application orchestration understand configured and actual repair counts before changing the embedded prompt policy. diff --git a/internal/app/app.go b/internal/app/app.go index 227560d..e7eac39 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -89,6 +89,7 @@ type ReportResult struct { ModelName string SourceWarnings []weatherdata.SourceWarning ValidationStatus promptexec.ValidationStatus + RepairAttempts *int LLMDebugPath string OutputPath string Notification *NotificationResult @@ -139,6 +140,7 @@ type BatchReportResult struct { ModelName string `json:"modelName,omitempty"` SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"` ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"` + RepairAttempts *int `json:"-"` LLMDebugPath string `json:"llmDebugPath,omitempty"` OutputPath string `json:"outputPath,omitempty"` } @@ -457,6 +459,9 @@ func copyBatchReportDetails(item *BatchReportResult, result *ReportResult) { item.Timezone = result.Timezone item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...) item.ValidationStatus = result.ValidationStatus + if result.RepairAttempts != nil { + item.RepairAttempts = repairAttemptsPointer(*result.RepairAttempts) + } } func batchInspectionCandidates(req BatchRequest, now time.Time) ([]report.Resolved, error) { diff --git a/internal/app/comparison_execution.go b/internal/app/comparison_execution.go index ff08858..e0d92c7 100644 --- a/internal/app/comparison_execution.go +++ b/internal/app/comparison_execution.go @@ -31,6 +31,7 @@ type comparisonProfileOutcome struct { ModelName string Status string ValidationStatus promptexec.ValidationStatus + RepairAttempts *int ReportPath string Markdown []byte LLMDebugPath string @@ -108,6 +109,9 @@ func executeComparisonProfile(ctx context.Context, req comparisonExecutionReques }) outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName outcome.ValidationStatus = execution.ValidationStatus + if execution.RepairAttempts != nil { + outcome.RepairAttempts = repairAttemptsPointer(*execution.RepairAttempts) + } outcome.LLMDebugPath = execution.LLMDebugPath if err != nil { outcome.canceled = cancellationError(err) diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go index 348ed26..9bfc834 100644 --- a/internal/app/generation_test.go +++ b/internal/app/generation_test.go @@ -68,6 +68,7 @@ type generationExecutor struct { beforeExecute func(promptexec.ExecuteRequest) cancelBeforeReturn context.CancelFunc validation promptexec.ValidationStatus + repairAttempts int validations map[string]promptexec.ValidationStatus rawOutput []byte waitForCancellation map[string]bool @@ -128,6 +129,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute profileErr := e.executeErrors[req.ProfileID] executeErr := e.executeErr status := e.validation + repairAttempts := e.repairAttempts if profileStatus, ok := e.validations[req.ProfileID]; ok { status = profileStatus } @@ -162,7 +164,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute if cancelBeforeReturn != nil { cancelBeforeReturn() } - execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil)} + execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", repairAttempts, nil)} if complete != nil { complete(execution) } diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go index 567d12c..84cf272 100644 --- a/internal/app/profile_execution.go +++ b/internal/app/profile_execution.go @@ -24,6 +24,7 @@ type profileExecutionOutcome struct { BackendID string ModelName string ValidationStatus promptexec.ValidationStatus + RepairAttempts *int LLMDebugPath string } @@ -99,6 +100,7 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p if execution == nil { return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)} } + outcome.RepairAttempts = repairAttemptsPointer(execution.Validation.RepairAttempts) if preparationCount != 1 { return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: promptProvenanceError()} } @@ -178,12 +180,19 @@ func validateExecutionProvenance(req profileExecutionRequest, preparation prompt if execution.PromptID != preparation.PromptID || execution.PromptVersion != preparation.PromptVersion || execution.PromptHash != preparation.PromptHash || execution.RenderedPromptHash != preparation.RenderedPromptHash || !reflect.DeepEqual(execution.InputHashes, preparation.InputHashes) || execution.ProfileID != preparation.ProfileID || execution.BackendID != preparation.BackendID || execution.ModelName != preparation.ModelName || - execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" { + execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" || + execution.Validation.RepairAttempts < 0 || execution.Validation.RepairAttempts > preparation.Output.RepairAttempts || + preparation.Output.RepairAttempts != definition.GeneratedTextRepairAttempts { return promptProvenanceError() } return nil } +func repairAttemptsPointer(value int) *int { + copy := value + return © +} + func promptProvenanceError() error { return promptexec.NewError(promptexec.InvalidConfiguration, "prompt execution provenance is inconsistent", nil) } diff --git a/internal/app/profile_execution_test.go b/internal/app/profile_execution_test.go index 0f5c178..286d66e 100644 --- a/internal/app/profile_execution_test.go +++ b/internal/app/profile_execution_test.go @@ -26,7 +26,7 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) { if err != nil { t.Fatalf("executePreparedProfile() error = %v", err) } - if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 { + if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 0 || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 { t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls) } if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) { @@ -34,6 +34,20 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) { } } +func TestExecutePreparedProfileRetainsCompletedRepairAttemptsOnLaterFailure(t *testing.T) { + prepared, inspection := preparedDailyProfile(t) + prepared.resolved.Definition.GeneratedTextRepairAttempts = 1 + executor := &generationExecutor{repairAttempts: 1, rawOutput: []byte(`{"summary":42}`), prepare: func(value *promptexec.Preparation) { value.Output.RepairAttempts = 1 }} + outcome, _, err := executePreparedProfile(context.Background(), profileExecutionRequest{ + Prepared: prepared, Prompt: inspection, + Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName}, + Executor: executor, + }) + if err == nil || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 1 { + t.Fatalf("outcome/error = %#v/%v", outcome, err) + } +} + func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) { prepared, inspection := preparedDailyProfile(t) debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir()) diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index 30542b5..77af8c5 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -48,6 +48,9 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report }) result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName result.ValidationStatus = outcome.ValidationStatus + if outcome.RepairAttempts != nil { + result.RepairAttempts = repairAttemptsPointer(*outcome.RepairAttempts) + } result.LLMDebugPath = outcome.LLMDebugPath if err != nil { return result, generatedProfileExecutionError(req.Resolved, result.RunID, err) diff --git a/internal/app/prompt_inspection.go b/internal/app/prompt_inspection.go index 42970a5..e43153b 100644 --- a/internal/app/prompt_inspection.go +++ b/internal/app/prompt_inspection.go @@ -207,7 +207,7 @@ func validPromptInput(inputs []promptexec.InputDefinition) bool { } func validPromptOutput(definition report.Definition, output promptexec.OutputContract) bool { - return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json" + return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json" && output.RepairAttempts == definition.GeneratedTextRepairAttempts } func promptInspectionError(operation string, err error) error { diff --git a/internal/report/daily_report.go b/internal/report/daily_report.go index 2e781fa..bbff680 100644 --- a/internal/report/daily_report.go +++ b/internal/report/daily_report.go @@ -9,14 +9,15 @@ import ( func dailyDefinition() Definition { return Definition{ - ID: Daily, - Name: "Daily Report", - PromptID: "weather.daily_generated_text", - PromptVersion: "2.0.0", - TemplateID: "daily", - GeneratedTextSchemaID: "daily", - ArtifactGroup: "daily", - OutputName: "daily.md", + ID: Daily, + Name: "Daily Report", + PromptID: "weather.daily_generated_text", + PromptVersion: "2.0.0", + TemplateID: "daily", + GeneratedTextSchemaID: "daily", + GeneratedTextRepairAttempts: 0, + ArtifactGroup: "daily", + OutputName: "daily.md", DistributorPathTemplates: []string{ "daily/{valid_start_date}/{run_id}.md", "daily/{valid_start_date}/index.md", diff --git a/internal/report/definition.go b/internal/report/definition.go index d579cca..3a800f6 100644 --- a/internal/report/definition.go +++ b/internal/report/definition.go @@ -27,20 +27,21 @@ const ( ) type Definition struct { - ID ID - Name string - PromptID string - PromptVersion string - TemplateID string - GeneratedTextSchemaID string - ArtifactGroup string - OutputName string - DistributorPathTemplates []string - Modules []module.ConfigItem - Morning bool - Evening bool - resolve func(ResolveRequest) (timeutil.Period, error) - runIDDisambiguator func(Resolved) string + ID ID + Name string + PromptID string + PromptVersion string + TemplateID string + GeneratedTextSchemaID string + GeneratedTextRepairAttempts int + ArtifactGroup string + OutputName string + DistributorPathTemplates []string + Modules []module.ConfigItem + Morning bool + Evening bool + resolve func(ResolveRequest) (timeutil.Period, error) + runIDDisambiguator func(Resolved) string } func (r Resolved) OutputName() (string, error) { diff --git a/internal/report/hourly_report.go b/internal/report/hourly_report.go index 17caf38..bf0e757 100644 --- a/internal/report/hourly_report.go +++ b/internal/report/hourly_report.go @@ -11,14 +11,15 @@ const hourlyReportHours = 6 func hourlyDefinition() Definition { return Definition{ - ID: Hourly, - Name: "Hourly Report", - PromptID: "weather.hourly_generated_text", - PromptVersion: "2.0.0", - TemplateID: "hourly", - GeneratedTextSchemaID: "hourly", - ArtifactGroup: "hourly", - OutputName: "hourly.md", + ID: Hourly, + Name: "Hourly Report", + PromptID: "weather.hourly_generated_text", + PromptVersion: "2.0.0", + TemplateID: "hourly", + GeneratedTextSchemaID: "hourly", + GeneratedTextRepairAttempts: 0, + ArtifactGroup: "hourly", + OutputName: "hourly.md", DistributorPathTemplates: []string{ "hourly/index.md", }, diff --git a/internal/report/period_test.go b/internal/report/period_test.go index 40028a4..382436c 100644 --- a/internal/report/period_test.go +++ b/internal/report/period_test.go @@ -91,6 +91,9 @@ func TestRegistryContainsOnlyPromptBackedReports(t *testing.T) { if definition.TemplateID == "" || definition.GeneratedTextSchemaID == "" { t.Fatalf("%s template/schema = %q/%q, want both set", definition.ID, definition.TemplateID, definition.GeneratedTextSchemaID) } + if definition.GeneratedTextRepairAttempts != 0 { + t.Fatalf("%s repair attempts = %d, want 0", definition.ID, definition.GeneratedTextRepairAttempts) + } } if _, err := registry.Lookup(ID("three_day")); err == nil { diff --git a/internal/report/today_report.go b/internal/report/today_report.go index 6e90703..9423339 100644 --- a/internal/report/today_report.go +++ b/internal/report/today_report.go @@ -7,14 +7,15 @@ import ( func todayDefinition() Definition { return Definition{ - ID: Today, - Name: "Today Report", - PromptID: "weather.today_generated_text", - PromptVersion: "2.0.0", - TemplateID: "today", - GeneratedTextSchemaID: "today", - ArtifactGroup: "today", - OutputName: "today.md", + ID: Today, + Name: "Today Report", + PromptID: "weather.today_generated_text", + PromptVersion: "2.0.0", + TemplateID: "today", + GeneratedTextSchemaID: "today", + GeneratedTextRepairAttempts: 0, + ArtifactGroup: "today", + OutputName: "today.md", DistributorPathTemplates: []string{ "daily/{valid_start_date}/{run_id}.md", "daily/{valid_start_date}/index.md", diff --git a/internal/report/tomorrow_report.go b/internal/report/tomorrow_report.go index 46ca6de..fdca49c 100644 --- a/internal/report/tomorrow_report.go +++ b/internal/report/tomorrow_report.go @@ -7,14 +7,15 @@ import ( func tomorrowDefinition() Definition { return Definition{ - ID: Tomorrow, - Name: "Tomorrow Report", - PromptID: "weather.tomorrow_generated_text", - PromptVersion: "2.0.0", - TemplateID: "tomorrow", - GeneratedTextSchemaID: "tomorrow", - ArtifactGroup: "tomorrow", - OutputName: "tomorrow.md", + ID: Tomorrow, + Name: "Tomorrow Report", + PromptID: "weather.tomorrow_generated_text", + PromptVersion: "2.0.0", + TemplateID: "tomorrow", + GeneratedTextSchemaID: "tomorrow", + GeneratedTextRepairAttempts: 0, + ArtifactGroup: "tomorrow", + OutputName: "tomorrow.md", DistributorPathTemplates: []string{ "daily/{valid_start_date}/{run_id}.md", "daily/{valid_start_date}/index.md",