From 97215ddb9b16640d7e5a8b480bc40cce6ca7bbf5 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 1 Aug 2026 20:06:40 +0000 Subject: [PATCH] Expand stateless workflow test coverage --- internal/app/batch_generation_test.go | 67 +++++++++++ internal/app/generation_test.go | 167 ++++++++++++++++++++++++-- internal/cli/result_test.go | 5 +- internal/cli/run_test.go | 36 ++++++ 4 files changed, 263 insertions(+), 12 deletions(-) create mode 100644 internal/app/batch_generation_test.go diff --git a/internal/app/batch_generation_test.go b/internal/app/batch_generation_test.go new file mode 100644 index 0000000..935397c --- /dev/null +++ b/internal/app/batch_generation_test.go @@ -0,0 +1,67 @@ +package app + +import ( + "context" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" +) + +func TestRunBatchDetailedKeepsSuccessfulOutputAndSkipsNotificationAfterPartialFailure(t *testing.T) { + bundle := generationBundle(t) + bundle.Hourly.Periods = bundle.Hourly.Periods[:1] + notifier := &generationNotifier{} + executor := &generationExecutor{failedPrompt: generationDefinitionForPrompt("weather.tomorrow_generated_text").PromptID} + result, err := RunBatchDetailed(context.Background(), BatchRequest{ + Config: generationDistributorConfig(), Batch: BatchMorning, + Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: t.TempDir(), + Collector: &generationCollector{bundle: &bundle}, Executor: executor, Notifier: notifier, + }) + if err != nil || result == nil || result.Total != 2 || result.Succeeded != 1 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "skipped" || notifier.batchCalls != 0 { + t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier) + } + if result.Reports[0].Status != "succeeded" || result.Reports[0].OutputPath == "" || result.Reports[1].Status != "failed" || result.Reports[1].OutputPath != "" { + t.Fatalf("report results = %#v", result.Reports) + } + if data, readErr := os.ReadFile(result.Reports[0].OutputPath); readErr != nil || len(data) == 0 { + t.Fatalf("successful output = %q, error = %v", data, readErr) + } +} + +func TestRunBatchDetailedNotifiesOnlyAfterAllOutputsExist(t *testing.T) { + bundle := generationBundle(t) + bundle.Hourly.Periods = bundle.Hourly.Periods[:1] + outputDir := t.TempDir() + notifier := &generationNotifier{} + result, err := RunBatchDetailed(context.Background(), BatchRequest{ + Config: generationDistributorConfig(), Batch: BatchMorning, + Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, + Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier, + }) + if err != nil || result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || notifier.batchCalls != 1 || result.Notification == nil || result.Notification.Status != "succeeded" { + t.Fatalf("RunBatchDetailed() result/error/notifier = %#v/%v/%#v", result, err, notifier) + } + if len(notifier.batchRequest.Files) < 2 || len(notifier.batchRequest.IncludedReports) != 2 { + t.Fatalf("batch notification = %#v", notifier.batchRequest) + } + if result.Reports[0].OutputPath == result.Reports[1].OutputPath { + t.Fatalf("batch reports share output path %q", result.Reports[0].OutputPath) + } + for _, file := range notifier.batchRequest.Files { + if filepath.Dir(file.SourcePath) != outputDir || file.BundlePath == "" { + t.Fatalf("notification file = %#v", file) + } + if _, statErr := os.Stat(file.SourcePath); statErr != nil { + t.Fatalf("notification source %q: %v", file.SourcePath, statErr) + } + } +} + +func generationDistributorConfig() config.Config { + cfg := generationConfig() + cfg.Notify.Distributor.Enabled = true + cfg.Notify.Distributor.PipelineIDTemplate = "weather" + return cfg +} diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go index 680376f..3d978b3 100644 --- a/internal/app/generation_test.go +++ b/internal/app/generation_test.go @@ -19,28 +19,67 @@ import ( type generationCollector struct { bundle *weatherdata.Bundle err error + called bool } -func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) { +func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) { + c.called = true return &collect.Result{Bundle: c.bundle}, c.err } -type generationExecutor struct{ called bool } +type generationExecutor struct { + called bool + inspectErr error + executeErr error + respectCancellation bool + validation promptexec.ValidationStatus + rawOutput []byte + failedPrompt string +} -func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { - definition := report.DefaultRegistry().MustLookup(report.Daily) +func (e generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { + if e.inspectErr != nil { + return promptexec.PromptInspection{}, e.inspectErr + } + definition := generationDefinitionForPrompt(id) return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil } func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil } -func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { +func (e *generationExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + if e.respectCancellation && ctx.Err() != nil { + return nil, ctx.Err() + } stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil { return nil, err } e.called = true - return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil + if e.executeErr != nil { + return nil, e.executeErr + } + status := e.validation + if status == "" { + status = promptexec.ValidationPassed + } + if e.failedPrompt == req.PromptID { + status = promptexec.ValidationFailed + } + rawOutput := e.rawOutput + if rawOutput == nil { + rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`) + } + return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil +} + +func generationDefinitionForPrompt(promptID string) report.Definition { + for _, definition := range report.DefaultRegistry().All() { + if definition.PromptID == promptID { + return definition + } + } + panic("unknown fixture prompt " + promptID) } func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) { @@ -49,15 +88,18 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) { bundle := generationBundle(t) executor := &generationExecutor{} workingDir := t.TempDir() - result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor}) + result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: executor}) if err != nil { t.Fatalf("GenerateDetailed() error = %v", err) } if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" { t.Fatalf("result = %#v", result) } + if result.LLMDebugPath != "" { + t.Fatalf("unexpected debug output = %q", result.LLMDebugPath) + } if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) { - t.Fatalf("workspace state = %v, want absent", err) + t.Fatalf("unexpected default state directory: %v", err) } data, err := os.ReadFile(result.OutputPath) if err != nil || len(data) == 0 { @@ -71,7 +113,7 @@ func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) collectionErr := errors.New("weather source unavailable") result, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), - WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{}, + WorkingDir: t.TempDir(), Collector: &generationCollector{err: collectionErr}, Executor: &generationExecutor{}, }) if !errors.Is(err, collectionErr) { t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr) @@ -81,6 +123,110 @@ func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) } } +func TestGenerateDetailedInspectsPromptBeforeCollectingWeather(t *testing.T) { + cfg := generationConfig() + inspectionErr := errors.New("profile is invalid") + collector := &generationCollector{bundle: generationBundlePointer(t)} + result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: collector, Executor: &generationExecutor{inspectErr: inspectionErr}}) + if !errors.Is(err, inspectionErr) || collector.called || result == nil { + t.Fatalf("GenerateDetailed() result/error/collector-called = %#v/%v/%t", result, err, collector.called) + } +} + +func TestGenerateDetailedPreservesDestinationBeforePublish(t *testing.T) { + for _, scenario := range []struct { + name string + executor generationExecutor + cancel bool + }{ + {name: "generation", executor: generationExecutor{executeErr: errors.New("provider unavailable")}}, + {name: "render", executor: generationExecutor{rawOutput: []byte(`{"summary":""}`)}}, + {name: "cancellation", executor: generationExecutor{respectCancellation: true}, cancel: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + outputPath := filepath.Join(t.TempDir(), "daily.md") + if err := os.WriteFile(outputPath, []byte("previous report"), 0o600); err != nil { + t.Fatal(err) + } + ctx := context.Background() + if scenario.cancel { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + bundle := generationBundle(t) + result, err := GenerateDetailed(ctx, GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &scenario.executor}) + data, readErr := os.ReadFile(outputPath) + if err == nil || result == nil || readErr != nil || string(data) != "previous report" { + t.Fatalf("GenerateDetailed() result/error/output = %#v/%v/%q (%v)", result, err, data, readErr) + } + }) + } +} + +func TestGenerateDetailedRetainsPublishedOutputWhenNotificationFails(t *testing.T) { + cfg := generationConfig() + cfg.Notify.Distributor.Enabled = true + cfg.Notify.Distributor.PipelineIDTemplate = "weather" + bundle := generationBundle(t) + outputPath := filepath.Join(t.TempDir(), "daily.md") + notifier := &generationNotifier{err: errors.New("distributor unavailable")} + result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier}) + if err == nil || result == nil || result.OutputPath != outputPath || notifier.request.ReportPath != outputPath || len(notifier.request.BundlePaths) == 0 { + t.Fatalf("GenerateDetailed() result/error/request = %#v/%v/%#v", result, err, notifier.request) + } + if data, readErr := os.ReadFile(outputPath); readErr != nil || len(data) == 0 { + t.Fatalf("published output = %q, error = %v", data, readErr) + } +} + +func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) { + bundle := generationBundle(t) + outputPath := filepath.Join(t.TempDir(), "daily.md") + if err := os.Mkdir(outputPath, 0o700); err != nil { + t.Fatal(err) + } + result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}}) + info, statErr := os.Stat(outputPath) + if err == nil || result == nil || statErr != nil || !info.IsDir() { + t.Fatalf("GenerateDetailed() result/error/output-info = %#v/%v/%#v (%v)", result, err, info, statErr) + } +} + +func generationConfig() config.Config { + cfg := config.Defaults() + cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home" + return cfg +} + +func generationBundlePointer(t *testing.T) *weatherdata.Bundle { + bundle := generationBundle(t) + return &bundle +} + +type generationNotifier struct { + err error + request NotificationRequest + batchRequest batchNotificationRequest + batchCalls int +} + +func (n *generationNotifier) Notify(_ context.Context, request NotificationRequest) (*NotificationResult, error) { + n.request = request + return nil, n.err +} + +func (n *generationNotifier) NotifyBatch(_ context.Context, request batchNotificationRequest) (*NotificationResult, error) { + n.batchCalls++ + n.batchRequest = request + for _, file := range request.Files { + if _, err := os.Stat(file.SourcePath); err != nil { + return nil, err + } + } + return &NotificationResult{Status: "succeeded", PipelineID: request.PipelineID, BundleID: request.BundleID}, nil +} + func generationBundle(t *testing.T) weatherdata.Bundle { t.Helper() data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) @@ -99,5 +245,6 @@ func generationTime(value string) time.Time { } var _ promptexec.Executor = (*generationExecutor)(nil) -var _ Collector = generationCollector{} +var _ Collector = (*generationCollector)(nil) +var _ Notifier = (*generationNotifier)(nil) var _ = report.Daily diff --git a/internal/cli/result_test.go b/internal/cli/result_test.go index b1c695f..7044b18 100644 --- a/internal/cli/result_test.go +++ b/internal/cli/result_test.go @@ -10,12 +10,13 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) func TestGenerateSummaryUsesActiveResultFields(t *testing.T) { generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) - summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil) - if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) { + summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil) + if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) || len(summary.SourceWarnings) != 1 { t.Fatalf("summary = %#v", summary) } data, err := json.Marshal(summary) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 5ba4575..5bc977c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -44,6 +44,42 @@ func TestResolveRunActionConstructsOneExecutor(t *testing.T) { } } +func TestResolveGenerateActionUsesInjectedWorkingDirectoryForOutputOverrides(t *testing.T) { + workingDir := t.TempDir() + configPath := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(configPath, []byte("weather_api:\n base_url: https://weather.api.example.com/\n"), 0o600); err != nil { + t.Fatal(err) + } + absoluteOutput := filepath.Join(t.TempDir(), "daily.md") + for _, scenario := range []struct { + name string + out string + want string + }{ + {name: "default", want: ""}, + {name: "relative", out: "reports/daily.md", want: filepath.Join(workingDir, "reports", "daily.md")}, + {name: "absolute", out: absoluteOutput, want: absoluteOutput}, + } { + t.Run(scenario.name, func(t *testing.T) { + runner := Runner{ + Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)}, + WorkingDir: workingDir, + ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { + return &factoryExecutor{}, nil + }, + } + args := []string{"daily", "--date", "2026-05-29", "--config", configPath} + if scenario.out != "" { + args = append(args, "--out", scenario.out) + } + req, _, err := runner.resolveGenerateAction(args) + if err != nil || req.WorkingDir != workingDir || req.OutputPath != scenario.want { + t.Fatalf("resolveGenerateAction() request/error = %#v/%v", req, err) + } + }) + } +} + func TestInspectCommandIsUnknownAndAbsentFromHelp(t *testing.T) { var stdout, stderr bytes.Buffer err := (Runner{}).Run(context.Background(), []string{"inspect", "reports"}, &stdout, &stderr)