eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Carry repair provenance through application workflows

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:46:56 +00:00
parent 20107b0dfd
commit ce79ea92c5
16 changed files with 101 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/app/app.go
View File

@@ -89,6 +89,7 @@ type ReportResult struct {
ModelName string
SourceWarnings []weatherdata.SourceWarning
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
LLMDebugPath string
OutputPath string
Notification *NotificationResult
@@ -139,6 +140,7 @@ type BatchReportResult struct {
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"-"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
}
@@ -457,6 +459,9 @@ func copyBatchReportDetails(item *BatchReportResult, result *ReportResult) {
item.Timezone = result.Timezone
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
item.ValidationStatus = result.ValidationStatus
if result.RepairAttempts != nil {
item.RepairAttempts = repairAttemptsPointer(*result.RepairAttempts)
}
}
func batchInspectionCandidates(req BatchRequest, now time.Time) ([]report.Resolved, error) {

4
internal/app/comparison_execution.go
View File

@@ -31,6 +31,7 @@ type comparisonProfileOutcome struct {
ModelName string
Status string
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
ReportPath string
Markdown []byte
LLMDebugPath string
@@ -108,6 +109,9 @@ func executeComparisonProfile(ctx context.Context, req comparisonExecutionReques
})
outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName
outcome.ValidationStatus = execution.ValidationStatus
if execution.RepairAttempts != nil {
outcome.RepairAttempts = repairAttemptsPointer(*execution.RepairAttempts)
}
outcome.LLMDebugPath = execution.LLMDebugPath
if err != nil {
outcome.canceled = cancellationError(err)

4
internal/app/generation_test.go
View File

@@ -68,6 +68,7 @@ type generationExecutor struct {
beforeExecute func(promptexec.ExecuteRequest)
cancelBeforeReturn context.CancelFunc
validation promptexec.ValidationStatus
repairAttempts int
validations map[string]promptexec.ValidationStatus
rawOutput []byte
waitForCancellation map[string]bool
@@ -128,6 +129,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
profileErr := e.executeErrors[req.ProfileID]
executeErr := e.executeErr
status := e.validation
repairAttempts := e.repairAttempts
if profileStatus, ok := e.validations[req.ProfileID]; ok {
status = profileStatus
}
@@ -162,7 +164,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
if cancelBeforeReturn != nil {
cancelBeforeReturn()
}
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil)}
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", repairAttempts, nil)}
if complete != nil {
complete(execution)
}

11
internal/app/profile_execution.go
View File

@@ -24,6 +24,7 @@ type profileExecutionOutcome struct {
BackendID string
ModelName string
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
LLMDebugPath string
}
@@ -99,6 +100,7 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p
if execution == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
}
outcome.RepairAttempts = repairAttemptsPointer(execution.Validation.RepairAttempts)
if preparationCount != 1 {
return outcome, nil, &profileExecutionError{operation: "validate prompt provenance", err: promptProvenanceError()}
}
@@ -178,12 +180,19 @@ func validateExecutionProvenance(req profileExecutionRequest, preparation prompt
if execution.PromptID != preparation.PromptID || execution.PromptVersion != preparation.PromptVersion || execution.PromptHash != preparation.PromptHash ||
execution.RenderedPromptHash != preparation.RenderedPromptHash || !reflect.DeepEqual(execution.InputHashes, preparation.InputHashes) ||
execution.ProfileID != preparation.ProfileID || execution.BackendID != preparation.BackendID || execution.ModelName != preparation.ModelName ||
execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" {
execution.Validation.Mode != "json_schema" || execution.Validation.SchemaPath != definition.GeneratedTextSchemaID+".generated_text.schema.json" ||
execution.Validation.RepairAttempts < 0 || execution.Validation.RepairAttempts > preparation.Output.RepairAttempts ||
preparation.Output.RepairAttempts != definition.GeneratedTextRepairAttempts {
return promptProvenanceError()
}
return nil
}
func repairAttemptsPointer(value int) *int {
copy := value
return &copy
}
func promptProvenanceError() error {
return promptexec.NewError(promptexec.InvalidConfiguration, "prompt execution provenance is inconsistent", nil)
}

16
internal/app/profile_execution_test.go
View File

@@ -26,7 +26,7 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
if err != nil {
t.Fatalf("executePreparedProfile() error = %v", err)
}
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 0 || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
}
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
@@ -34,6 +34,20 @@ func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
}
}
func TestExecutePreparedProfileRetainsCompletedRepairAttemptsOnLaterFailure(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
prepared.resolved.Definition.GeneratedTextRepairAttempts = 1
executor := &generationExecutor{repairAttempts: 1, rawOutput: []byte(`{"summary":42}`), prepare: func(value *promptexec.Preparation) { value.Output.RepairAttempts = 1 }}
outcome, _, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor,
})
if err == nil || outcome.RepairAttempts == nil || *outcome.RepairAttempts != 1 {
t.Fatalf("outcome/error = %#v/%v", outcome, err)
}
}
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())

3
internal/app/prompt_generate.go
View File

@@ -48,6 +48,9 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
})
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
result.ValidationStatus = outcome.ValidationStatus
if outcome.RepairAttempts != nil {
result.RepairAttempts = repairAttemptsPointer(*outcome.RepairAttempts)
}
result.LLMDebugPath = outcome.LLMDebugPath
if err != nil {
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)

2
internal/app/prompt_inspection.go
View File

@@ -207,7 +207,7 @@ func validPromptInput(inputs []promptexec.InputDefinition) bool {
}
func validPromptOutput(definition report.Definition, output promptexec.OutputContract) bool {
return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json"
return output.Format == "json" && output.ValidationMode == "json_schema" && output.SchemaPath == definition.GeneratedTextSchemaID+".generated_text.schema.json" && output.RepairAttempts == definition.GeneratedTextRepairAttempts
}
func promptInspectionError(operation string, err error) error {

17
internal/report/daily_report.go
View File

@@ -9,14 +9,15 @@ import (
func dailyDefinition() Definition {
return Definition{
ID: Daily,
Name: "Daily Report",
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
TemplateID: "daily",
GeneratedTextSchemaID: "daily",
ArtifactGroup: "daily",
OutputName: "daily.md",
ID: Daily,
Name: "Daily Report",
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
TemplateID: "daily",
GeneratedTextSchemaID: "daily",
GeneratedTextRepairAttempts: 0,
ArtifactGroup: "daily",
OutputName: "daily.md",
DistributorPathTemplates: []string{
"daily/{valid_start_date}/{run_id}.md",
"daily/{valid_start_date}/index.md",

29
internal/report/definition.go
View File

@@ -27,20 +27,21 @@ const (
)
type Definition struct {
ID ID
Name string
PromptID string
PromptVersion string
TemplateID string
GeneratedTextSchemaID string
ArtifactGroup string
OutputName string
DistributorPathTemplates []string
Modules []module.ConfigItem
Morning bool
Evening bool
resolve func(ResolveRequest) (timeutil.Period, error)
runIDDisambiguator func(Resolved) string
ID ID
Name string
PromptID string
PromptVersion string
TemplateID string
GeneratedTextSchemaID string
GeneratedTextRepairAttempts int
ArtifactGroup string
OutputName string
DistributorPathTemplates []string
Modules []module.ConfigItem
Morning bool
Evening bool
resolve func(ResolveRequest) (timeutil.Period, error)
runIDDisambiguator func(Resolved) string
}
func (r Resolved) OutputName() (string, error) {

17
internal/report/hourly_report.go
View File

@@ -11,14 +11,15 @@ const hourlyReportHours = 6
func hourlyDefinition() Definition {
return Definition{
ID: Hourly,
Name: "Hourly Report",
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
TemplateID: "hourly",
GeneratedTextSchemaID: "hourly",
ArtifactGroup: "hourly",
OutputName: "hourly.md",
ID: Hourly,
Name: "Hourly Report",
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
TemplateID: "hourly",
GeneratedTextSchemaID: "hourly",
GeneratedTextRepairAttempts: 0,
ArtifactGroup: "hourly",
OutputName: "hourly.md",
DistributorPathTemplates: []string{
"hourly/index.md",
},

3
internal/report/period_test.go
View File

@@ -91,6 +91,9 @@ func TestRegistryContainsOnlyPromptBackedReports(t *testing.T) {
if definition.TemplateID == "" || definition.GeneratedTextSchemaID == "" {
t.Fatalf("%s template/schema = %q/%q, want both set", definition.ID, definition.TemplateID, definition.GeneratedTextSchemaID)
}
if definition.GeneratedTextRepairAttempts != 0 {
t.Fatalf("%s repair attempts = %d, want 0", definition.ID, definition.GeneratedTextRepairAttempts)
}
}
if _, err := registry.Lookup(ID("three_day")); err == nil {

17
internal/report/today_report.go
View File

@@ -7,14 +7,15 @@ import (
func todayDefinition() Definition {
return Definition{
ID: Today,
Name: "Today Report",
PromptID: "weather.today_generated_text",
PromptVersion: "2.0.0",
TemplateID: "today",
GeneratedTextSchemaID: "today",
ArtifactGroup: "today",
OutputName: "today.md",
ID: Today,
Name: "Today Report",
PromptID: "weather.today_generated_text",
PromptVersion: "2.0.0",
TemplateID: "today",
GeneratedTextSchemaID: "today",
GeneratedTextRepairAttempts: 0,
ArtifactGroup: "today",
OutputName: "today.md",
DistributorPathTemplates: []string{
"daily/{valid_start_date}/{run_id}.md",
"daily/{valid_start_date}/index.md",

17
internal/report/tomorrow_report.go
View File

@@ -7,14 +7,15 @@ import (
func tomorrowDefinition() Definition {
return Definition{
ID: Tomorrow,
Name: "Tomorrow Report",
PromptID: "weather.tomorrow_generated_text",
PromptVersion: "2.0.0",
TemplateID: "tomorrow",
GeneratedTextSchemaID: "tomorrow",
ArtifactGroup: "tomorrow",
OutputName: "tomorrow.md",
ID: Tomorrow,
Name: "Tomorrow Report",
PromptID: "weather.tomorrow_generated_text",
PromptVersion: "2.0.0",
TemplateID: "tomorrow",
GeneratedTextSchemaID: "tomorrow",
GeneratedTextRepairAttempts: 0,
ArtifactGroup: "tomorrow",
OutputName: "tomorrow.md",
DistributorPathTemplates: []string{
"daily/{valid_start_date}/{run_id}.md",
"daily/{valid_start_date}/index.md",