eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Enable PromptKit repair attempts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:49:57 +00:00
parent ce79ea92c5
commit 0c9cd6d5fb
23 changed files with 61 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/adapters/promptkit/adapter_test.go
View File

@@ -126,11 +126,11 @@ func (client *fakeClient) allRequests() []promptkit.GenerateRequest {
func TestInspectPromptAndProfile(t *testing.T) {
adapter := newTestAdapter(t, &fakeClient{})
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "2.0.0")
inspection, err := adapter.InspectPrompt(context.Background(), "weather.daily_generated_text", "2.1.0")
if err != nil {
t.Fatalf("InspectPrompt() error = %v", err)
}
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != "weather-balanced" {
if inspection.PromptID != "weather.daily_generated_text" || inspection.PromptVersion != "2.1.0" || inspection.DefaultProfileID != "weather-balanced" || inspection.Output.RepairAttempts != 1 {
t.Fatalf("inspection = %#v", inspection)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "data_package" || !inspection.Inputs[0].Required || inspection.Inputs[0].ContentType != "application/yaml" {
@@ -404,7 +404,7 @@ func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
}
request := promptexec.ExecuteRequest{
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
}
@@ -832,7 +832,7 @@ func writeProfileFile(t *testing.T, profile string) string {
func testExecuteRequest() promptexec.ExecuteRequest {
return promptexec.ExecuteRequest{
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
}

2
internal/app/app.go
View File

@@ -140,7 +140,7 @@ type BatchReportResult struct {
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"-"`
RepairAttempts *int `json:"repairAttempts,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
}

3
internal/app/comparison_execution_test.go
View File

@@ -170,7 +170,8 @@ func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteReq
e.mu.Lock()
profile := e.profiles[req.ProfileID]
e.mu.Unlock()
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID + ".generated_text.schema.json"}, StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
definition := generationDefinitionForPrompt(req.PromptID)
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json", RepairAttempts: definition.GeneratedTextRepairAttempts}, StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
e.callbackFailures <- err
return nil, err
}

5
internal/app/generation_test.go
View File

@@ -89,7 +89,7 @@ func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string
return promptexec.PromptInspection{}, e.inspectErr
}
definition := generationDefinitionForPrompt(id)
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json", RepairAttempts: definition.GeneratedTextRepairAttempts}}, nil
}
func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
generationExecutorMu.Lock()
@@ -113,7 +113,8 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
calls = 1
}
for range calls {
preparation := promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID + ".generated_text.schema.json"}, StartedAt: stamp, EndedAt: stamp}
definition := generationDefinitionForPrompt(req.PromptID)
preparation := promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json", RepairAttempts: definition.GeneratedTextRepairAttempts}, StartedAt: stamp, EndedAt: stamp}
if prepare != nil {
prepare(&preparation)
}

2
internal/app/prompt_inspection_test.go
View File

@@ -350,7 +350,7 @@ func validPromptInspection(definition report.Definition) promptexec.PromptInspec
return promptexec.PromptInspection{
PromptID: definition.PromptID, PromptVersion: definition.PromptVersion, PromptHash: "prompt-hash", DefaultProfileID: "default-profile",
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json", RepairAttempts: definition.GeneratedTextRepairAttempts},
}
}

2
internal/cli/comparison_test.go
View File

@@ -355,7 +355,7 @@ func comparisonResult(outputDirectory string, results []app.ComparisonProfileRes
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),

2
internal/cli/generate_test.go
View File

@@ -132,7 +132,7 @@ func actionConfigPath(t *testing.T) string {
func generatedReportResult() *app.ReportResult {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
return &app.ReportResult{
ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0",
RunID: "daily-20260529", GeneratedAt: generatedAt, Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)},
ProfileID: "weather-light", BackendID: "local", ModelName: "weather-model", ValidationStatus: promptexec.ValidationPassed,

5
internal/cli/result.go
View File

@@ -43,6 +43,7 @@ type generateSummary struct {
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus string `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"repairAttempts,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
}
@@ -129,6 +130,10 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
summary.ValidationStatus = string(result.ValidationStatus)
if result.RepairAttempts != nil {
value := *result.RepairAttempts
summary.RepairAttempts = &value
}
summary.OutputPath = result.OutputPath
summary.LLMDebugPath = result.LLMDebugPath
summary.Notification = newGenerateNotificationSummary(result.Notification)

6
internal/cli/result_test.go
View File

@@ -19,7 +19,7 @@ import (
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) || len(summary.SourceWarnings) != 1 {
t.Fatalf("summary = %#v", summary)
}
@@ -39,7 +39,7 @@ func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
result := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
@@ -113,7 +113,7 @@ func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
failure := comparison.NewSafeError("generation", "execute prompt failed")
complete := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.1.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
Total: 2, Succeeded: 2,

3
internal/promptassets/assets/prompts/daily/daily_generated_text.yml
View File

@@ -1,5 +1,5 @@
id: weather.daily_generated_text
version: "2.0.0"
version: "2.1.0"
default_profile: weather-balanced
description: Daily weather report analysis prompt.
inputs:
@@ -21,3 +21,4 @@ output:
format: json
validation_mode: json_schema
schema_path: daily.generated_text.schema.json
repair_attempts: 1

3
internal/promptassets/assets/prompts/hourly/hourly_generated_text.yml
View File

@@ -1,5 +1,5 @@
id: weather.hourly_generated_text
version: "2.0.0"
version: "2.1.0"
default_profile: weather-light
description: Hourly weather report analysis prompt.
inputs:
@@ -21,3 +21,4 @@ output:
format: json
validation_mode: json_schema
schema_path: hourly.generated_text.schema.json
repair_attempts: 1

3
internal/promptassets/assets/prompts/today/today_generated_text.yml
View File

@@ -1,5 +1,5 @@
id: weather.today_generated_text
version: "2.0.0"
version: "2.1.0"
default_profile: weather-balanced
description: Today's weather report analysis prompt.
inputs:
@@ -21,3 +21,4 @@ output:
format: json
validation_mode: json_schema
schema_path: today.generated_text.schema.json
repair_attempts: 1

3
internal/promptassets/assets/prompts/tomorrow/tomorrow_generated_text.yml
View File

@@ -1,5 +1,5 @@
id: weather.tomorrow_generated_text
version: "2.0.0"
version: "2.1.0"
default_profile: weather-balanced
description: Tomorrow's weather report analysis prompt.
inputs:
@@ -21,3 +21,4 @@ output:
format: json
validation_mode: json_schema
schema_path: tomorrow.generated_text.schema.json
repair_attempts: 1

14
internal/promptassets/promptassets_test.go
View File

@@ -77,8 +77,8 @@ func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) {
if err := yaml.Unmarshal(data, &definition); err != nil {
t.Fatalf("decode prompt definition: %v", err)
}
if definition.ID != tc.id || definition.Version != "2.0.0" || definition.DefaultProfile != tc.profile {
t.Fatalf("definition = %#v, want %s version 2.0.0 and profile %s", definition, tc.id, tc.profile)
if definition.ID != tc.id || definition.Version != "2.1.0" || definition.DefaultProfile != tc.profile {
t.Fatalf("definition = %#v, want %s version 2.1.0 and profile %s", definition, tc.id, tc.profile)
}
sharedInstruction := false
for _, message := range definition.Messages {
@@ -92,8 +92,8 @@ func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) {
if len(definition.Inputs) != 1 || definition.Inputs[0].Name != "data_package" || !definition.Inputs[0].Required || definition.Inputs[0].ContentType != "application/yaml" {
t.Fatalf("inputs = %#v, want one required YAML data_package", definition.Inputs)
}
if definition.Output.Format != "json" || definition.Output.ValidationMode != "json_schema" || definition.Output.SchemaPath != tc.schemaID+".generated_text.schema.json" || definition.Output.RepairAttempts != nil {
t.Fatalf("output = %#v, want JSON schema output without repair attempts", definition.Output)
if definition.Output.Format != "json" || definition.Output.ValidationMode != "json_schema" || definition.Output.SchemaPath != tc.schemaID+".generated_text.schema.json" || definition.Output.RepairAttempts == nil || *definition.Output.RepairAttempts != 1 {
t.Fatalf("output = %#v, want JSON schema output with one repair attempt", definition.Output)
}
if _, err := promptassets.Schema(tc.schemaID); err != nil {
t.Fatalf("Schema(%q) error = %v", tc.schemaID, err)
@@ -286,11 +286,11 @@ func TestPromptkitInspectsEmbeddedPromptsOffline(t *testing.T) {
{"weather.hourly_generated_text", "weather-light", "deepseek/deepseek-v4-flash"},
} {
t.Run(want.id, func(t *testing.T) {
inspection, err := engine.InspectPrompt(context.Background(), want.id, "2.0.0")
inspection, err := engine.InspectPrompt(context.Background(), want.id, "2.1.0")
if err != nil {
t.Fatalf("InspectPrompt() error = %v", err)
}
if inspection.PromptID != want.id || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != want.profile {
if inspection.PromptID != want.id || inspection.PromptVersion != "2.1.0" || inspection.DefaultProfileID != want.profile || inspection.OutputContract.RepairAttempts != 1 {
t.Fatalf("inspection = %#v", inspection)
}
profile, err := engine.InspectProfile(context.Background(), inspection.DefaultProfileID)
@@ -414,7 +414,7 @@ func TestPromptAssetsExcludeRetiredRuntimeSettings(t *testing.T) {
if err != nil {
return err
}
for _, unwanted := range []string{"local-heavy", "pipeline-weather/", "application/json", "repair_attempts:", "weather.daily_report"} {
for _, unwanted := range []string{"local-heavy", "pipeline-weather/", "application/json", "weather.daily_report"} {
if strings.Contains(string(data), unwanted) {
t.Fatalf("%s contains retired runtime setting %q", path, unwanted)
}

4
internal/report/daily_report.go
View File

@@ -12,10 +12,10 @@ func dailyDefinition() Definition {
ID: Daily,
Name: "Daily Report",
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
TemplateID: "daily",
GeneratedTextSchemaID: "daily",
GeneratedTextRepairAttempts: 0,
GeneratedTextRepairAttempts: 1,
ArtifactGroup: "daily",
OutputName: "daily.md",
DistributorPathTemplates: []string{

4
internal/report/hourly_report.go
View File

@@ -14,10 +14,10 @@ func hourlyDefinition() Definition {
ID: Hourly,
Name: "Hourly Report",
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
TemplateID: "hourly",
GeneratedTextSchemaID: "hourly",
GeneratedTextRepairAttempts: 0,
GeneratedTextRepairAttempts: 1,
ArtifactGroup: "hourly",
OutputName: "hourly.md",
DistributorPathTemplates: []string{

8
internal/report/period_test.go
View File

@@ -82,8 +82,8 @@ func TestRegistryContainsOnlyPromptBackedReports(t *testing.T) {
}
for _, definition := range definitions {
if definition.PromptVersion != "2.0.0" {
t.Fatalf("%s PromptVersion = %q, want 2.0.0", definition.ID, definition.PromptVersion)
if definition.PromptVersion != "2.1.0" {
t.Fatalf("%s PromptVersion = %q, want 2.1.0", definition.ID, definition.PromptVersion)
}
if definition.PromptID == "" {
t.Fatalf("%s PromptID is empty", definition.ID)
@@ -91,8 +91,8 @@ func TestRegistryContainsOnlyPromptBackedReports(t *testing.T) {
if definition.TemplateID == "" || definition.GeneratedTextSchemaID == "" {
t.Fatalf("%s template/schema = %q/%q, want both set", definition.ID, definition.TemplateID, definition.GeneratedTextSchemaID)
}
if definition.GeneratedTextRepairAttempts != 0 {
t.Fatalf("%s repair attempts = %d, want 0", definition.ID, definition.GeneratedTextRepairAttempts)
if definition.GeneratedTextRepairAttempts != 1 {
t.Fatalf("%s repair attempts = %d, want 1", definition.ID, definition.GeneratedTextRepairAttempts)
}
}

4
internal/report/today_report.go
View File

@@ -10,10 +10,10 @@ func todayDefinition() Definition {
ID: Today,
Name: "Today Report",
PromptID: "weather.today_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
TemplateID: "today",
GeneratedTextSchemaID: "today",
GeneratedTextRepairAttempts: 0,
GeneratedTextRepairAttempts: 1,
ArtifactGroup: "today",
OutputName: "today.md",
DistributorPathTemplates: []string{

4
internal/report/tomorrow_report.go
View File

@@ -10,10 +10,10 @@ func tomorrowDefinition() Definition {
ID: Tomorrow,
Name: "Tomorrow Report",
PromptID: "weather.tomorrow_generated_text",
PromptVersion: "2.0.0",
PromptVersion: "2.1.0",
TemplateID: "tomorrow",
GeneratedTextSchemaID: "tomorrow",
GeneratedTextRepairAttempts: 0,
GeneratedTextRepairAttempts: 1,
ArtifactGroup: "tomorrow",
OutputName: "tomorrow.md",
DistributorPathTemplates: []string{