eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Carry repair provenance through application workflows

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:46:56 +00:00
parent 20107b0dfd
commit ce79ea92c5
16 changed files with 101 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/app/app.go
View File

@@ -89,6 +89,7 @@ type ReportResult struct {
ModelName string
SourceWarnings []weatherdata.SourceWarning
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
LLMDebugPath string
OutputPath string
Notification *NotificationResult
@@ -139,6 +140,7 @@ type BatchReportResult struct {
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"-"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
}
@@ -457,6 +459,9 @@ func copyBatchReportDetails(item *BatchReportResult, result *ReportResult) {
item.Timezone = result.Timezone
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
item.ValidationStatus = result.ValidationStatus
if result.RepairAttempts != nil {
item.RepairAttempts = repairAttemptsPointer(*result.RepairAttempts)
}
}
func batchInspectionCandidates(req BatchRequest, now time.Time) ([]report.Resolved, error) {

4
internal/app/comparison_execution.go
View File

@@ -31,6 +31,7 @@ type comparisonProfileOutcome struct {
ModelName string
Status string
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
ReportPath string
Markdown []byte
LLMDebugPath string
@@ -108,6 +109,9 @@ func executeComparisonProfile(ctx context.Context, req comparisonExecutionReques
})
outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName
outcome.ValidationStatus = execution.ValidationStatus
if execution.RepairAttempts != nil {
outcome.RepairAttempts = repairAttemptsPointer(*execution.RepairAttempts)
}
outcome.LLMDebugPath = execution.LLMDebugPath
if err != nil {
outcome.canceled = cancellationError(err)

4
internal/app/generation_test.go
View File

@@ -68,6 +68,7 @@ type generationExecutor struct {
beforeExecute func(promptexec.ExecuteRequest)
cancelBeforeReturn context.CancelFunc
validation promptexec.ValidationStatus
repairAttempts int
validations map[string]promptexec.ValidationStatus
rawOutput []byte
waitForCancellation map[string]bool
@@ -128,6 +129,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
profileErr := e.executeErrors[req.ProfileID]
executeErr := e.executeErr
status := e.validation
repairAttempts := e.repairAttempts
if profileStatus, ok := e.validations[req.ProfileID]; ok {
status = profileStatus
}
@@ -162,7 +164,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
if cancelBeforeReturn != nil {
cancelBeforeReturn()
}
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil)}
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", repairAttempts, nil)}
if complete != nil {
complete(execution)
}

11
internal/app/profile_execution.go
View File

@@ -24,6 +24,7 @@ type profileExecutionOutcome struct {
BackendID string
ModelName string
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
LLMDebugPath string
}
@@ -99,6 +100,7 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p
if execution == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
}
outcome.RepairAttempts = repairAttemptsPointer(execution.Validation.RepairAttempts)
if preparationCount != 1 {
return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: promptProvenanceError()}
}
@@ -178,12 +180,19 @@ func validateExecutionProvenance(req profileExecutionRequest, preparation prompt
if execution.PromptID != preparation.PromptID || execution.PromptVersion != preparation.PromptVersion || execution.PromptHash != preparation.PromptHash ||
execution.RenderedPromptHash != preparation.RenderedPromptHash || !reflect.DeepEqual(execution.InputHashes, preparation.InputHashes) ||
execution.ProfileID != preparation.ProfileID || execution.BackendID != preparation.BackendID || execution.ModelName != preparation.ModelName ||
execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" {
execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" ||
execution.Validation.RepairAttempts < 0 || execution.Validation.RepairAttempts > preparation.Output.RepairAttempts ||
preparation.Output.RepairAttempts != definition.GeneratedTextRepairAttempts {
return promptProvenanceError()
}
return nil
}
func repairAttemptsPointer(value int) *int {
copy := value
return &copy
}
func promptProvenanceError() error {
return promptexec.NewError(promptexec.InvalidConfiguration, "prompt execution provenance is inconsistent", nil)
}

16
internal/app/profile_execution_test.go
View File

@@ -26,7 +26,7 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
if err != nil {
t.Fatalf("executePreparedProfile() error = %v", err)
}
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 0 || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
}
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
@@ -34,6 +34,20 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
}
}
func TestExecutePreparedProfileRetainsCompletedRepairAttemptsOnLaterFailure(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
prepared.resolved.Definition.GeneratedTextRepairAttempts = 1
executor := &generationExecutor{repairAttempts: 1, rawOutput: []byte(`{"summary":42}`), prepare: func(value *promptexec.Preparation) { value.Output.RepairAttempts = 1 }}
outcome, _, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor,
})
if err == nil || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 1 {
t.Fatalf("outcome/error = %#v/%v", outcome, err)
}
}
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())

3
internal/app/prompt_generate.go
View File

@@ -48,6 +48,9 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
})
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
result.ValidationStatus = outcome.ValidationStatus
if outcome.RepairAttempts != nil {
result.RepairAttempts = repairAttemptsPointer(*outcome.RepairAttempts)
}
result.LLMDebugPath = outcome.LLMDebugPath
if err != nil {
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)

2
internal/app/prompt_inspection.go
View File

@@ -207,7 +207,7 @@ func validPromptInput(inputs []promptexec.InputDefinition) bool {
}
func validPromptOutput(definition report.Definition, output promptexec.OutputContract) bool {
return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json"
return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json" && output.RepairAttempts == definition.GeneratedTextRepairAttempts
}
func promptInspectionError(operation string, err error) error {