package app import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) type assembledBatchExecutor struct { definitions map[string]report.Definition promptRequests []string profileRequests []string executeRequests []promptexec.ExecuteRequest failures map[int]error active int maxActive int } func newAssembledBatchExecutor() *assembledBatchExecutor { definitions := make(map[string]report.Definition) for _, definition := range report.DefaultRegistry().All() { definitions[definition.PromptID] = definition } return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)} } func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { e.promptRequests = append(e.promptRequests, id+"@"+version) definition, ok := e.definitions[id] if !ok || definition.PromptVersion != version { return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection") } return logicalPromptInspection(definition), nil } func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { e.profileRequests = append(e.profileRequests, id) return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil } func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { call := len(e.executeRequests) e.executeRequests = append(e.executeRequests, req) e.active++ if e.active > e.maxActive { e.maxActive = e.active } defer func() { e.active-- }() stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) preparation := promptexec.Preparation{ PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, } if err := callback(preparation, nil); err != nil { return nil, err } if err := e.failures[call]; err != nil { return nil, err } definition := e.definitions[req.PromptID] return &promptexec.Execution{ RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash", StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: []byte(generatedTextForPrompt(req.PromptID)), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil), }, nil } type assembledBatchNotifier struct { reportRequests []NotificationRequest batchRequests []batchNotificationRequest batchResult *NotificationResult batchErr error } func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) { n.reportRequests = append(n.reportRequests, req) return nil, errors.New("per-report notification must be suppressed") } func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) { n.batchRequests = append(n.batchRequests, req) if n.batchErr != nil { return nil, n.batchErr } if n.batchResult != nil { result := *n.batchResult if result.PipelineID == "" { result.PipelineID = req.PipelineID } if result.BundleID == "" { result.BundleID = req.BundleID } if result.IdempotencyKey == "" { result.IdempotencyKey = req.IdempotencyKey } return &result, nil } return &NotificationResult{ RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted", }, nil } func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) { tests := []struct { name string batch BatchKind now time.Time wantIDs []report.ID wantCopies []string }{ {name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}}, {name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := assembledBatchConfig(t, false) bundle := assembledBatchBundle(t, "2026-05-31") collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} executor := newAssembledBatchExecutor() outputDir := filepath.Join(t.TempDir(), "output") debugRoot := filepath.Join(t.TempDir(), "debug") result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: test.batch, Now: test.now, OutputDir: outputDir, LLMDebugDir: debugRoot, Collector: collector, Executor: executor, }) if err != nil { t.Fatalf("RunBatchDetailed() error = %v", err) } if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 { t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed) } if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 { t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs)) } if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" { t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests) } for index, item := range result.Reports { if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" { t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index]) } if executor.executeRequests[index].PromptID != item.PromptID { t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID) } if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" { t.Fatalf("successful report paths = %#v", item) } assertBatchItemMatchesMetadata(t, item) if filepath.Base(item.OutputPath) != test.wantCopies[index] { t.Fatalf("output copy = %q, want %q", item.OutputPath, test.wantCopies[index]) } assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath) managed, readErr := os.ReadFile(item.ReportPath) if readErr != nil { t.Fatalf("read managed report: %v", readErr) } copied, readErr := os.ReadFile(item.OutputPath) if readErr != nil || !bytes.Equal(managed, copied) { t.Fatalf("output copy mismatch/error = %v", readErr) } } }) } } func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) { cfg := assembledBatchConfig(t, true) bundle := assembledBatchBundle(t, "2026-05-31") collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} executor := newAssembledBatchExecutor() executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil) notifier := &assembledBatchNotifier{} result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"), Collector: collector, Executor: executor, Notifier: notifier, }) if err != nil { t.Fatalf("RunBatchDetailed() error = %v", err) } if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 { t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed) } if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" { t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification) } for index, item := range result.Reports { if index == 1 { if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) { t.Fatalf("failed item = %#v", item) } if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" { t.Fatalf("failed item reached paths = %#v", item) } assertBatchItemMatchesMetadata(t, item) continue } if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" { t.Fatalf("continued item %d = %#v", index, item) } } } func TestRunBatchDetailedNotificationLifecycle(t *testing.T) { t.Run("disabled", func(t *testing.T) { cfg := assembledBatchConfig(t, false) bundle := assembledBatchBundle(t, "2026-05-31") notifier := &assembledBatchNotifier{} result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, }) if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 { t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests)) } }) t.Run("batch disabled", func(t *testing.T) { cfg := assembledBatchConfig(t, true) cfg.Notify.Distributor.Batch.Enabled = false bundle := assembledBatchBundle(t, "2026-05-31") notifier := &assembledBatchNotifier{} result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, }) if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 { t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests)) } }) t.Run("all success", func(t *testing.T) { cfg := assembledBatchConfig(t, true) bundle := assembledBatchBundle(t, "2026-05-31") notifier := &assembledBatchNotifier{batchResult: &NotificationResult{ RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted", Report: []byte(`{"actions":[{"action":"replace_older"}]}`), }} outputDir := filepath.Join(t.TempDir(), "output") result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), OutputDir: outputDir, Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, }) if err != nil { t.Fatalf("RunBatchDetailed() error = %v", err) } if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || result.Notification.Path == "" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 { t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests)) } managedPaths := make(map[string]struct{}, len(result.Reports)) for _, item := range result.Reports { managedPaths[item.ReportPath] = struct{}{} if item.NotificationPath != "" { t.Fatalf("report item contains per-report notification path: %#v", item) } } request := notifier.batchRequests[0] if len(request.IncludedReports) != len(result.Reports) { t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports)) } for _, file := range request.Files { if _, ok := managedPaths[file.SourcePath]; !ok || strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" { t.Fatalf("notification file = %#v, want managed Markdown source", file) } } artifact := readBatchNotificationArtifact(t, result.Notification.Path) if artifact.Status != "succeeded" || artifact.Upload == nil || artifact.Upload.RunID != "batch-notification-run" || artifact.RunStatus == nil || len(artifact.Reports) != len(result.Reports) { t.Fatalf("notification artifact = %#v", artifact) } }) t.Run("upload failure", func(t *testing.T) { cfg := assembledBatchConfig(t, true) bundle := assembledBatchBundle(t, "2026-05-31") notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")} result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, }) if err != nil { t.Fatalf("RunBatchDetailed() error = %v", err) } if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" || result.Notification.Path == "" { t.Fatalf("result = %#v, want successful reports and failed notification", result) } for _, item := range result.Reports { if item.Status != "succeeded" { t.Fatalf("report item = %#v, want success despite notification failure", item) } } artifact := readBatchNotificationArtifact(t, result.Notification.Path) if artifact.Status != "failed" || !strings.Contains(artifact.Error, "batch upload rejected") { t.Fatalf("notification artifact = %#v", artifact) } }) t.Run("status report", func(t *testing.T) { cfg := assembledBatchConfig(t, true) bundle := assembledBatchBundle(t, "2026-05-31") notifier := &assembledBatchNotifier{batchResult: &NotificationResult{ RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted", StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`), }} result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, }) if err != nil || result.Failed != 0 || result.Notification == nil || result.Notification.Path == "" { t.Fatalf("result/error = %#v/%v", result, err) } artifact := readBatchNotificationArtifact(t, result.Notification.Path) if artifact.StatusError != "status lookup unavailable" || artifact.RunStatus == nil || !bytes.Contains(artifact.RunStatus.Report, []byte("replace_older")) { t.Fatalf("notification artifact = %#v", artifact) } }) } func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) { cfg := assembledBatchConfig(t, false) bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01") outputDir := filepath.Join(t.TempDir(), "output") debugRoot := filepath.Join(t.TempDir(), "debug") result, err := RunBatchDetailed(context.Background(), BatchRequest{ Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), OutputDir: outputDir, LLMDebugDir: debugRoot, Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), }) if err != nil || result.Failed != 0 || len(result.Reports) != 3 { t.Fatalf("result/error = %#v/%v", result, err) } seenRuns := make(map[string]struct{}) seenDebug := make(map[string]struct{}) dailyDates := make(map[string]BatchReportResult) for _, item := range result.Reports { if _, exists := seenRuns[item.RunID]; exists { t.Fatalf("duplicate run ID %q", item.RunID) } seenRuns[item.RunID] = struct{}{} if _, exists := seenDebug[item.LLMDebugPath]; exists { t.Fatalf("duplicate debug path %q", item.LLMDebugPath) } seenDebug[item.LLMDebugPath] = struct{}{} if item.ReportID == report.Daily { date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02") dailyDates[date] = item } } for _, date := range []string{"2026-05-31", "2026-06-01"} { item, ok := dailyDates[date] if !ok { t.Fatalf("daily items = %#v, want %s", dailyDates, date) } if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") { t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath) } wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID) if item.LLMDebugPath != wantDebugPrefix { t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix) } assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json")) } } func assembledBatchConfig(t *testing.T, notify bool) config.Config { t.Helper() cfg := workflowConfig(t) cfg.Notify.Distributor.Enabled = notify cfg.Notify.Distributor.Batch.Enabled = notify return cfg } func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle { t.Helper() bundle := workflowBundle(t) location := mustLoadTestLocation(t, "America/Chicago") for _, date := range dates { periods := fullDayPeriods(t, date, location) for index := range periods { temperature := float64(60 + index) periods[index].TemperatureF = &temperature periods[index].TextDescription = "Partly cloudy" } bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...) } return bundle } func generatedTextForPrompt(promptID string) string { switch promptID { case "weather.today_generated_text": return validTodayWorkflowJSON() case "weather.tomorrow_generated_text": return validTomorrowWorkflowJSON() case "weather.daily_generated_text": return validDailyWorkflowJSON() case "weather.hourly_generated_text": return validHourlyWorkflowJSON() default: return "" } } func assertBatchPathsExist(t *testing.T, paths ...string) { t.Helper() for _, path := range paths { if _, err := os.Stat(path); err != nil { t.Fatalf("expected path %q: %v", path, err) } } } func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) { t.Helper() data, err := os.ReadFile(item.MetadataPath) if err != nil { t.Fatalf("read metadata %q: %v", item.MetadataPath, err) } var metadata state.Metadata if err := json.Unmarshal(data, &metadata); err != nil { t.Fatalf("decode metadata %q: %v", item.MetadataPath, err) } if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID || item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath || item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath || item.NotificationPath != metadata.NotificationPath { t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata) } } func readBatchNotificationArtifact(t *testing.T, path string) state.BatchDistributorNotificationArtifact { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read batch notification artifact: %v", err) } var artifact state.BatchDistributorNotificationArtifact if err := json.Unmarshal(data, &artifact); err != nil { t.Fatalf("decode batch notification artifact: %v", err) } return artifact } func loadBatchDataPackage(t *testing.T, path string) promptinput.Package { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read data package: %v", err) } pkg, err := promptinput.LoadYAML(data) if err != nil { t.Fatalf("LoadYAML() error = %v", err) } return pkg }