package app import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) type workflowCollector struct { result *collect.Result err error calls int } func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) { c.calls++ if c.err != nil { return nil, c.err } return c.result, nil } type workflowExecutor struct { definition report.Definition raw []byte prompt promptexec.PromptInspection inspectionErr error profile promptexec.ProfileInspection profileErr error beforePreparationErr error afterCallbackErr error afterPreparationErr error validation promptexec.ValidationStatus executeCalls int providerCalls int request promptexec.ExecuteRequest beforeProvider func() preparationDebug *promptexec.PreparationDebug executionDebug *promptexec.ExecutionDebug preparation *promptexec.Preparation execution *promptexec.Execution } func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { if e.inspectionErr != nil { return promptexec.PromptInspection{}, e.inspectionErr } if id != e.definition.PromptID || version != e.definition.PromptVersion { return promptexec.PromptInspection{}, errors.New("unexpected prompt identity") } if e.prompt.PromptID != "" { return e.prompt, nil } return validPromptInspection(e.definition), nil } func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { if e.profileErr != nil { return promptexec.ProfileInspection{}, e.profileErr } profile := e.profile if profile.ProfileID == "" { profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"} } return profile, nil } func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { e.executeCalls++ e.request = req if e.beforePreparationErr != nil { return nil, e.beforePreparationErr } stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) profile := e.profile if profile.ProfileID == "" { profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"} } preparation := promptexec.Preparation{ PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, } e.preparation = &preparation if err := callback(preparation, e.preparationDebug); err != nil { return nil, err } if e.afterCallbackErr != nil { return nil, e.afterCallbackErr } if e.beforeProvider != nil { e.beforeProvider() } e.providerCalls++ if e.afterPreparationErr != nil { return nil, e.afterPreparationErr } validation := e.validation if validation == "" { validation = promptexec.ValidationPassed } execution := &promptexec.Execution{ RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash", StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw, Debug: e.executionDebug, Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil), } e.execution = execution return execution, nil } type workflowNotifier struct { requests []NotificationRequest err error } func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) { n.requests = append(n.requests, req) if n.err != nil { return nil, n.err } return &NotificationResult{ RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID, IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted", }, nil } func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) { tests := []struct { name string kind ReportKind id report.ID date time.Time raw string wantOutput string }{ {name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."}, {name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."}, {name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."}, {name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) definition := report.DefaultRegistry().MustLookup(test.id) bundle := workflowBundle(t) collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)} notifier := &workflowNotifier{} outputPath := filepath.Join(t.TempDir(), test.name+".md") result, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier, }) if err != nil { t.Fatalf("GenerateDetailed() error = %v", err) } if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID { t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID) } if executor.request.PromptVersion != definition.PromptVersion { t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion) } filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace) if storeErr != nil { t.Fatalf("NewFilesystemStore() error = %v", storeErr) } preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath) if loadErr != nil || preparation.PromptVersion != definition.PromptVersion { t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion) } if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 { t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls) } persisted, readErr := os.ReadFile(result.DataPackagePath) if readErr != nil { t.Fatalf("read data package: %v", readErr) } if !bytes.Equal(executor.request.DataPackage, persisted) { t.Fatal("executor data package differs from exact persisted YAML bytes") } renderedReport, readErr := os.ReadFile(result.ReportPath) if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) { t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput) } copied, readErr := os.ReadFile(outputPath) if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath { t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath) } if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath { t.Fatalf("notification requests = %#v, want selected output source", notifier.requests) } wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup if notifier.requests[0].PipelineID != wantPipeline { t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline) } validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02") wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID) if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") { t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths) } renderedName := filepath.Base(result.ReportPath) if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" { t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath) } }) } } func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) { tests := []struct { name string kind ReportKind id report.ID raw string override string profile promptexec.ProfileInspection }{ { name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"}, }, { name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"}, }, { name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile", profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) cfg.Promptkit.Profile = test.override definition := report.DefaultRegistry().MustLookup(test.id) executor := &workflowExecutor{ definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw), } bundle := workflowBundle(t) _, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{}, }) if err != nil { t.Fatalf("GenerateDetailed() error = %v", err) } if executor.request.ProfileID != test.profile.ProfileID { t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID) } if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName { t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName) } if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName { t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName) } }) } } type preparationFailingStore struct { state.Store } type renderContextFailingStore struct { state.Store } func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) { path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot) if err != nil { return "", err } for index := range snapshot.Outputs { snapshot.Outputs[index].Value = "invalid module value" } return path, nil } func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) { return "", errors.New("injected preparation persistence failure") } func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) { tests := []struct { name string configure func(*workflowExecutor) wantCategory promptexec.ErrorCategory wantPreparation bool wantExecution bool wantRaw bool wantProviderCall int }{ {name: "preparation", configure: func(e *workflowExecutor) { e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil) }, wantCategory: promptexec.Generation, wantPreparation: true}, {name: "credential disappears", configure: func(e *workflowExecutor) { e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil) }, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true}, {name: "capacity is not retried", configure: func(e *workflowExecutor) { e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil) }, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "canceled", configure: func(e *workflowExecutor) { e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled) }, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "deadline", configure: func(e *workflowExecutor) { e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded) }, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "generation", configure: func(e *workflowExecutor) { e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil) }, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "operational validation error", configure: func(e *workflowExecutor) { e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil) }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, {name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) cfg.Notify.Distributor.Enabled = false definition := report.DefaultRegistry().MustLookup(report.Daily) executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())} test.configure(executor) bundle := workflowBundle(t) collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} result, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: collector, Executor: executor, }) if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory { t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory) } if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw { t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath) } if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall { t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall) } }) } } func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) { tests := []struct { name string configure func(*workflowExecutor) wantCategory promptexec.ErrorCategory }{ {name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration}, {name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration}, {name: "malformed profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret") }, wantCategory: promptexec.InvalidConfiguration}, {name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration}, {name: "credential", configure: func(e *workflowExecutor) { e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true} }, wantCategory: promptexec.MissingCredential}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) definition := report.DefaultRegistry().MustLookup(report.Daily) executor := &workflowExecutor{definition: definition} test.configure(executor) collector := &workflowCollector{err: errors.New("collector must not run")} result, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: collector, Executor: executor, }) if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 { t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls) } if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") { t.Fatalf("error leaks profile details: %v", err) } entries, readErr := os.ReadDir(cfg.Workspace.Root) if readErr != nil || len(entries) != 0 { t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr) } }) } } func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) { cfg := workflowConfig(t) cfg.Notify.Distributor.Enabled = false filesystem, err := state.NewFilesystemStore(cfg.Workspace) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } definition := report.DefaultRegistry().MustLookup(report.Daily) executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())} bundle := workflowBundle(t) result, err := GenerateDetailed(context.Background(), GenerateRequest{ Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem}, }) if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 { t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls) } } func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) { cfg := workflowConfig(t) cfg.Notify.Distributor.Enabled = false now := workflowTime("2026-05-29T08:30:00-05:00") request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()} resolved, err := ResolveGenerate(request, now) if err != nil { t.Fatalf("ResolveGenerate() error = %v", err) } filesystem, err := state.NewFilesystemStore(cfg.Workspace) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } paths, err := filesystem.Paths(resolved) if err != nil { t.Fatalf("Paths() error = %v", err) } checked := false executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())} executor.beforeProvider = func() { checked = true if _, statErr := os.Stat(paths.Preparation); statErr != nil { t.Fatalf("preparation was not durable before provider execution: %v", statErr) } } bundle := workflowBundle(t) request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}} request.Executor = executor request.Store = filesystem result, err := GenerateDetailed(context.Background(), request) if err != nil || result == nil || !checked || result.OutputPath == "" { t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath) } } func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) { tests := []struct { name string raw string configure func(*GenerateRequest, *workflowNotifier) wantRaw bool wantNormalized bool wantContext bool wantReport bool wantOutput bool }{ {name: "generated text decode", raw: `{`, wantRaw: true}, {name: "generated text domain", raw: `{}`, wantRaw: true}, {name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { req.Store = renderContextFailingStore{Store: req.Store} }, wantRaw: true, wantNormalized: true}, {name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext} }, wantRaw: true, wantNormalized: true}, {name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { blocker := filepath.Join(t.TempDir(), "report-blocker") if err := os.Mkdir(blocker, 0o700); err != nil { t.Fatalf("create report blocker: %v", err) } req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker} }, wantRaw: true, wantNormalized: true, wantContext: true}, {name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) { notifier.err = errors.New("notification rejected") }, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) definition := report.DefaultRegistry().MustLookup(report.Daily) executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)} notifier := &workflowNotifier{} bundle := workflowBundle(t) filesystem, err := state.NewFilesystemStore(cfg.Workspace) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } req := GenerateRequest{ Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: notifier, Store: filesystem, } if test.configure != nil { test.configure(&req, notifier) } result, err := GenerateDetailed(context.Background(), req) if err == nil || result == nil { t.Fatalf("result/error = %#v/%v, want partial result and error", result, err) } if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized || (result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport || (result.OutputPath != "") != test.wantOutput { t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath) } if test.wantRaw { persisted, readErr := os.ReadFile(result.GeneratedTextRawPath) if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) { t.Fatalf("retained raw output = %q, error %v", persisted, readErr) } } if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) { t.Fatalf("notification requests = %#v", notifier.requests) } if test.name == "notification" && result.Metadata.NotificationPath != "" { t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata) } }) } } func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) { tests := []struct { name string createCollision func(string, report.Resolved) error wantProviderCalls int wantPreparationFile bool }{ { name: "preparation debug", createCollision: func(root string, resolved report.Resolved) error { path := workflowDebugRunPath(root, resolved) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return err } return os.WriteFile(path, []byte("not a directory"), 0o600) }, }, { name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true, createCollision: func(root string, resolved report.Resolved) error { return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700) }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := workflowConfig(t) cfg.Notify.Distributor.Enabled = false now := workflowTime("2026-05-29T08:30:00-05:00") request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()} resolved, err := ResolveGenerate(request, now) if err != nil { t.Fatalf("ResolveGenerate() error = %v", err) } debugRoot := filepath.Join(t.TempDir(), "prompt-debug") if err := test.createCollision(debugRoot, resolved); err != nil { t.Fatalf("create debug collision: %v", err) } bundle := workflowBundle(t) executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())} request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}} request.Executor = executor request.LLMDebugDir = debugRoot result, err := GenerateDetailed(context.Background(), request) if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err)) } if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" { t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath) } if test.wantPreparationFile && result.MetadataPath == "" { t.Fatal("execution debug failure lost previously persisted metadata") } preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json") _, statErr := os.Stat(preparationDebug) if (statErr == nil) != test.wantPreparationFile { t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile) } }) } } func workflowDebugRunPath(root string, resolved report.Resolved) string { return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID) } func workflowBundlePaths(id report.ID, validDate, runID string) []string { switch id { case report.Daily: return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"} case report.Today: return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"} case report.Tomorrow: return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"} case report.Hourly: return []string{"hourly/index.md"} default: return nil } } func workflowConfig(t *testing.T) config.Config { t.Helper() cfg := config.Defaults() cfg.Workspace.Root = t.TempDir() cfg.WeatherAPI.Timezone = "America/Chicago" cfg.Location.ID = "home" cfg.Location.Name = "Testville" cfg.Location.Region = "MO" cfg.Notify.Distributor.Enabled = true cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}" return cfg } func workflowBundle(t *testing.T) weatherdata.Bundle { t.Helper() data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) if err != nil { t.Fatalf("read bundle fixture: %v", err) } var bundle weatherdata.Bundle if err := json.Unmarshal(data, &bundle); err != nil { t.Fatalf("decode bundle fixture: %v", err) } future := bundle.Hourly.Periods[0] future.StartTime = workflowTime("2026-05-30T06:00:00-05:00") future.EndTime = workflowTime("2026-05-30T07:00:00-05:00") bundle.Hourly.Periods = append(bundle.Hourly.Periods, future) futureNarrative := bundle.Narrative.Periods[0] futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00") futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00") futureNarrative.Name = "Tomorrow" bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative) return bundle } func workflowTime(value string) time.Time { parsed, err := time.Parse(time.RFC3339, value) if err != nil { panic(err) } return parsed } func validHourlyWorkflowJSON() string { return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}` } func validTomorrowWorkflowJSON() string { return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}` } func validTodayWorkflowJSON() string { return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}` } func validDailyWorkflowJSON() string { return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}` }