package app import ( "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) const ( failPromptExecution = "prompt execution" failMetadata = "metadata" failGeneratedText = "generated text" failRenderContext = "render context" failRenderedReportPath = "rendered report path" failDistributorNotification = "distributor notification" ) type failingPersistenceStore struct { state.Store failOperation string failExecutionCall int failMetadataCall int executionCalls int metadataCalls int renderedReportPath string } func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) { s.executionCalls++ if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) { return "", errors.New("injected prompt execution persistence failure") } return s.Store.SavePromptExecution(ctx, resolved, artifact) } func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) { if s.failOperation == failGeneratedText { return "", errors.New("injected generated text persistence failure") } return s.Store.SaveGeneratedText(ctx, resolved, data) } func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) { if s.failOperation == failRenderContext { return "", errors.New("injected render context persistence failure") } return s.Store.SaveRenderContext(ctx, resolved, value) } func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) { if s.failOperation == failRenderedReportPath { return s.renderedReportPath, nil } return s.Store.PrepareRenderedReport(ctx, resolved) } func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) { if s.failOperation == failDistributorNotification { return "", errors.New("injected notification persistence failure") } return s.Store.SaveDistributorNotification(ctx, resolved, artifact) } func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) { s.metadataCalls++ if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall { return "", errors.New("injected metadata persistence failure") } return s.Store.SaveMetadata(ctx, metadata) } type artifactPathExecutor struct { beforePreparationErr error afterPreparationErr error validation promptexec.ValidationStatus } func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { return promptexec.PromptInspection{}, errors.New("unexpected inspection") } func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) { return promptexec.ProfileInspection{}, errors.New("unexpected inspection") } func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { if e.beforePreparationErr != nil { return nil, e.beforePreparationErr } now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) if err := callback(promptexec.Preparation{ PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test", ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now, }, nil); err != nil { return nil, err } if e.afterPreparationErr != nil { return nil, e.afterPreparationErr } validation := e.validation if validation == "" { validation = promptexec.ValidationPassed } return &promptexec.Execution{ RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil), }, nil } type successfulNotifier struct{} func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil } type failingNotifier struct{} func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { return nil, errors.New("injected notification failure") } func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { tests := []struct { name string failOperation string failMetadataCall int outputCopy bool notify bool want reachedPromptArtifacts }{ {name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}}, {name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}}, {name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}}, {name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}}, {name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}}, {name: "managed report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}}, {name: "output copy then metadata", failOperation: failMetadata, failMetadataCall: 5, outputCopy: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}}, {name: "notification then metadata", failOperation: failMetadata, failMetadataCall: 6, notify: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, notification: true}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { req, paths := promptArtifactRequest(t, artifactPathExecutor{}) store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall} req.Store = store if test.outputCopy { req.OutputPath = filepath.Join(t.TempDir(), "daily.md") paths.output = req.OutputPath } if test.notify { req.Config.Notify.Distributor.Enabled = true req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" req.Notifier = successfulNotifier{} req.noNotify = false } result, err := generatePromptReport(context.Background(), req) if err == nil || result == nil { t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) } assertReachedPromptArtifacts(t, result, paths, test.want) }) } } func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) { tests := []struct { name string executor artifactPathExecutor want reachedPromptArtifacts wantExecutionStatus state.PromptExecutionStatus wantExecutionPaths state.PromptExecutionPaths wantRawExecution bool }{ { name: "preparation failure", executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)}, want: reachedPromptArtifacts{preparation: true, metadata: true}, }, { name: "operational execution failure", executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)}, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true}, wantExecutionStatus: state.PromptExecutionFailed, }, { name: "completed validation rejection", executor: artifactPathExecutor{validation: promptexec.ValidationFailed}, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, wantExecutionStatus: state.PromptExecutionValidationRejected, wantRawExecution: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { req, paths := promptArtifactRequest(t, test.executor) if test.wantRawExecution { test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw } result, err := generatePromptReport(context.Background(), req) if err == nil || result == nil { t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) } assertReachedPromptArtifacts(t, result, paths, test.want) if test.wantExecutionStatus != "" { artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath) if loadErr != nil { t.Fatalf("LoadPromptExecution() error = %v", loadErr) } if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths { t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths) } } }) } } func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) { req, paths := promptArtifactRequest(t, artifactPathExecutor{}) req.OutputPath = filepath.Join(t.TempDir(), "daily.md") paths.output = req.OutputPath req.Config.Notify.Distributor.Enabled = true req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" req.Notifier = successfulNotifier{} req.noNotify = false result, err := generatePromptReport(context.Background(), req) if err != nil { t.Fatalf("generatePromptReport() error = %v", err) } want := state.PromptExecutionPaths{ RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText, RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport, OutputPath: paths.output, NotificationPath: paths.Notification, } assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want) data, err := os.ReadFile(result.ExecutionPath) if err != nil { t.Fatalf("read execution artifact: %v", err) } text := string(data) for _, forbidden := range []string{ "Showers are possible during the selected day", `"rawOutput":`, `"debug":`, `"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`, "credential", "secret-value", } { if strings.Contains(text, forbidden) { t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text) } } } func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) { tests := []struct { name string failOperation string failExecutionCall int failMetadataCall int requestOutput bool failOutputCopy bool notify bool notificationFailure bool wantExecution reachedExecutionArtifacts wantResult reachedPromptArtifacts }{ { name: "normalized text write", failOperation: failGeneratedText, wantExecution: reachedExecutionArtifacts{raw: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, }, { name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2, wantExecution: reachedExecutionArtifacts{raw: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, }, { name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, }, { name: "render context write", failOperation: failRenderContext, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, }, { name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, }, { name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, }, { name: "managed report write", failOperation: failRenderedReportPath, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, }, { name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, }, { name: "output copy write", requestOutput: true, failOutputCopy: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, }, { name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, }, { name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, }, { name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, }, { name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, }, { name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, }, { name: "notification operation", requestOutput: true, notify: true, notificationFailure: true, wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { req, paths := promptArtifactRequest(t, artifactPathExecutor{}) store := &failingPersistenceStore{ Store: req.Store, failOperation: test.failOperation, failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall, } if test.failOperation == failRenderedReportPath { store.renderedReportPath = t.TempDir() } req.Store = store if test.requestOutput { req.OutputPath = filepath.Join(t.TempDir(), "daily.md") paths.output = req.OutputPath } if test.failOutputCopy { blocker := filepath.Join(t.TempDir(), "not-a-directory") if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil { t.Fatalf("write output blocker: %v", err) } req.OutputPath = filepath.Join(blocker, "daily.md") paths.output = req.OutputPath } if test.notify { req.Config.Notify.Distributor.Enabled = true req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" req.Notifier = successfulNotifier{} req.noNotify = false } if test.notificationFailure { req.Notifier = failingNotifier{} } result, err := generatePromptReport(context.Background(), req) if err == nil || result == nil { t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) } assertReachedPromptArtifacts(t, result, paths, test.wantResult) assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution)) }) } } type reachedExecutionArtifacts struct { raw bool normalized bool renderContext bool report bool output bool notification bool } func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths { result := state.PromptExecutionPaths{} if reached.raw { result.RawOutputPath = paths.GeneratedTextRaw } if reached.normalized { result.GeneratedTextPath = paths.GeneratedText } if reached.renderContext { result.RenderContextPath = paths.RenderContext } if reached.report { result.RenderedReportPath = paths.RenderedReport } if reached.output { result.OutputPath = paths.output } if reached.notification { result.NotificationPath = paths.Notification } return result } func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) { t.Helper() artifact, err := store.LoadPromptExecution(context.Background(), path) if err != nil { t.Fatalf("LoadPromptExecution() error = %v", err) } if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed { t.Fatalf("execution outcome changed after downstream write: %#v", artifact) } if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" { t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance) } if artifact.Paths != want { t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want) } } type promptArtifactPaths struct { state.ArtifactPaths output string } type reachedPromptArtifacts struct { preparation bool execution bool metadata bool raw bool normalized bool renderContext bool report bool output bool notification bool } func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) { t.Helper() cfg := config.Defaults() cfg.Workspace.Root = t.TempDir() resolved, err := ResolveGenerate(GenerateRequest{ Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), }, mustParse("2026-05-29T05:00:00-05:00")) if err != nil { t.Fatalf("ResolveGenerate() error = %v", err) } bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) if err != nil { t.Fatalf("read daily fixture: %v", err) } var bundle weatherdata.Bundle if err := json.Unmarshal(bundleData, &bundle); err != nil { t.Fatalf("decode daily fixture: %v", err) } filesystemStore, err := state.NewFilesystemStore(cfg.Workspace) if err != nil { t.Fatalf("NewFilesystemStore() error = %v", err) } paths, err := filesystemStore.Paths(resolved) if err != nil { t.Fatalf("Paths() error = %v", err) } debugWriter, err := state.NewPromptDebugWriter("") if err != nil { t.Fatalf("NewPromptDebugWriter() error = %v", err) } return promptReportRequest{ GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore}, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, Inspection: PromptInspectionResult{ PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model", }, DebugWriter: debugWriter, noNotify: true, }, promptArtifactPaths{ArtifactPaths: paths} } func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) { t.Helper() if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage { t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage) } if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata { t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata) } checks := []struct { name string got string metadataGot string inMetadata bool path string want bool }{ {"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation}, {"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution}, {"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata}, {"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw}, {"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized}, {"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext}, {"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report}, {"output", result.OutputPath, "", false, paths.output, want.output}, {"notification", result.NotificationPath, result.Metadata.NotificationPath, true, paths.Notification, want.notification}, } for _, check := range checks { if check.want && check.got != check.path { t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path) } if check.want && check.inMetadata && check.metadataGot != check.path { t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path) } if !check.want && check.got != "" { t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got) } if !check.want && check.inMetadata && check.metadataGot != "" { t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot) } } }