diff --git a/internal/adapters/promptkit/adapter.go b/internal/adapters/promptkit/adapter.go index b698f18..4f2ca4d 100644 --- a/internal/adapters/promptkit/adapter.go +++ b/internal/adapters/promptkit/adapter.go @@ -124,9 +124,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR PromptID: request.PromptID, PromptVersion: request.PromptVersion, ProfileID: request.ProfileID, - Inputs: map[string]promptkit.ArtifactRef{ - "data_package": promptkit.InlineWithURI(request.DataPackagePath, string(append([]byte(nil), request.DataPackage...))), - }, + Inputs: map[string]promptkit.ArtifactRef{"data_package": promptkit.Inline(string(append([]byte(nil), request.DataPackage...)))}, }) if err != nil { return nil, classifyError(err) @@ -134,7 +132,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR defer prepared.Discard() details := prepared.Details() - preparation, debug := preparationValues(details, request.DataPackagePath, request.CaptureDebug) + preparation, debug := preparationValues(details, request.CaptureDebug) if preparedCallback != nil { if err := preparedCallback(preparation, debug); err != nil { return nil, err @@ -145,7 +143,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR if err != nil { return nil, classifyError(err) } - return executionValue(result, request.DataPackagePath, request.CaptureDebug), nil + return executionValue(result, request.CaptureDebug), nil } func outputContract(value promptkit.OutputContract) promptexec.OutputContract { @@ -156,7 +154,7 @@ func outputContract(value promptkit.OutputContract) promptexec.OutputContract { } } -func preparationValues(value promptkit.PreparedRun, dataPackagePath string, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) { +func preparationValues(value promptkit.PreparedRun, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) { preparation := promptexec.Preparation{ PromptID: value.PromptID, PromptVersion: value.PromptVersion, @@ -170,7 +168,6 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt StartedAt: value.StartTime, EndedAt: value.EndTime, Duration: time.Duration(value.DurationMS) * time.Millisecond, - DataPackagePath: dataPackagePath, } if !captureDebug { return preparation, nil @@ -186,7 +183,7 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt return preparation, debug } -func executionValue(value *promptkit.RunResult, dataPackagePath string, captureDebug bool) *promptexec.Execution { +func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.Execution { if value == nil { return nil } @@ -214,12 +211,11 @@ func executionValue(value *promptkit.RunResult, dataPackagePath string, captureD CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens, }, - StartedAt: value.StartTime, - EndedAt: value.EndTime, - Duration: value.Duration, - Validation: validation, - DataPackagePath: dataPackagePath, - RawOutput: []byte(value.RawOutput), + StartedAt: value.StartTime, + EndedAt: value.EndTime, + Duration: value.Duration, + Validation: validation, + RawOutput: []byte(value.RawOutput), } if captureDebug { execution.Debug = &promptexec.ExecutionDebug{ diff --git a/internal/adapters/promptkit/adapter_test.go b/internal/adapters/promptkit/adapter_test.go index 59f6131..f60dc17 100644 --- a/internal/adapters/promptkit/adapter_test.go +++ b/internal/adapters/promptkit/adapter_test.go @@ -210,7 +210,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) { callbackCalls := 0 result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { callbackCalls++ - if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.DataPackagePath != request.DataPackagePath || preparation.ModelName != "test-model" { + if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" { t.Fatalf("preparation = %#v", preparation) } if debug != nil { @@ -227,7 +227,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) { if callbackCalls != 1 || client.callCount() != 1 { t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount()) } - if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content || result.DataPackagePath != request.DataPackagePath { + if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content { t.Fatalf("result = %#v", result) } if result.Debug != nil { @@ -250,11 +250,10 @@ func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) { t.Fatalf("newAdapter() error = %v", err) } request := promptexec.ExecuteRequest{ - PromptID: "weather.hourly_generated_text", - PromptVersion: "2.0.0", - ProfileID: "weather-light", - DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"), - DataPackagePath: "data-packages/hourly/data_package.yaml", + PromptID: "weather.hourly_generated_text", + PromptVersion: "2.0.0", + ProfileID: "weather-light", + DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"), } var preparation promptexec.Preparation prepared := false @@ -288,7 +287,7 @@ func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) { if _, err := adapter.Execute(context.Background(), request, nil); err != nil { t.Fatalf("Execute() error = %v", err) } - if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != request.DataPackagePath || reader.ref.Body != string(request.DataPackage) { + if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) { t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref) } } @@ -527,11 +526,10 @@ func writeProfileFile(t *testing.T, profile string) string { func testExecuteRequest() promptexec.ExecuteRequest { return promptexec.ExecuteRequest{ - PromptID: "weather.daily_generated_text", - PromptVersion: "2.0.0", - ProfileID: "test-profile", - DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"), - DataPackagePath: "data-packages/daily/data_package.yaml", + PromptID: "weather.daily_generated_text", + PromptVersion: "2.0.0", + ProfileID: "test-profile", + DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"), } } diff --git a/internal/app/app.go b/internal/app/app.go index 8d31818..01c8bde 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -21,7 +21,6 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) @@ -53,7 +52,6 @@ type GenerateRequest struct { Collector Collector Notifier Notifier Executor promptexec.Executor - Store state.Store } type BatchRequest struct { @@ -65,7 +63,6 @@ type BatchRequest struct { LLMDebugDir string Collector Collector Executor promptexec.Executor - Store state.Store Notifier Notifier } @@ -85,21 +82,22 @@ type ReportFacts struct { } type ReportResult struct { - ModuleSnapshot module.Snapshot - ModuleSnapshotPath string - DataPackage promptinput.Package - DataPackagePath string - PreparationPath string - ExecutionPath string - LLMDebugPath string - ReportPath string - OutputPath string - Metadata state.Metadata - MetadataPath string - GeneratedTextRawPath string - GeneratedTextPath string - RenderContextPath string - Notification *NotificationResult + ReportID report.ID + ReportName string + PromptID string + PromptVersion string + RunID string + GeneratedAt time.Time + Timezone string + ValidPeriod timeutil.Period + ProfileID string + BackendID string + ModelName string + SourceWarnings []weatherdata.SourceWarning + ValidationStatus promptexec.ValidationStatus + LLMDebugPath string + OutputPath string + Notification *NotificationResult } type BatchResult struct { @@ -132,25 +130,26 @@ type BatchNotificationReport struct { } type BatchReportResult struct { - ReportID report.ID `json:"reportId"` - ReportName string `json:"reportName"` - PromptID string `json:"promptId"` - RunID string `json:"runId"` - Status string `json:"status"` - Error string `json:"error,omitempty"` - NotificationStatus string `json:"notificationStatus,omitempty"` - NotificationRunID string `json:"notificationRunId,omitempty"` - NotificationPipelineID string `json:"notificationPipelineId,omitempty"` - NotificationError string `json:"notificationError,omitempty"` - GeneratedAt time.Time `json:"generatedAt"` - ValidPeriod timeutil.Period `json:"validPeriod"` - DataPackagePath string `json:"dataPackagePath,omitempty"` - PreparationPath string `json:"preparationPath,omitempty"` - ExecutionPath string `json:"executionPath,omitempty"` - LLMDebugPath string `json:"llmDebugPath,omitempty"` - ReportPath string `json:"reportPath,omitempty"` - OutputPath string `json:"outputPath,omitempty"` - MetadataPath string `json:"metadataPath,omitempty"` + ReportID report.ID `json:"reportId"` + ReportName string `json:"reportName"` + PromptID string `json:"promptId"` + RunID string `json:"runId"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + NotificationStatus string `json:"notificationStatus,omitempty"` + NotificationRunID string `json:"notificationRunId,omitempty"` + NotificationPipelineID string `json:"notificationPipelineId,omitempty"` + NotificationError string `json:"notificationError,omitempty"` + GeneratedAt time.Time `json:"generatedAt"` + ValidPeriod timeutil.Period `json:"validPeriod"` + Timezone string `json:"timezone"` + ProfileID string `json:"profileId,omitempty"` + BackendID string `json:"backendId,omitempty"` + ModelName string `json:"modelName,omitempty"` + SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"` + ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"` + LLMDebugPath string `json:"llmDebugPath,omitempty"` + OutputPath string `json:"outputPath,omitempty"` } type BatchError struct { @@ -260,14 +259,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, if err != nil { return nil, err } + result := initialReportResult(req, resolved, PromptInspectionResult{}) outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved) if err != nil { - return nil, err + return result, err } req.OutputPath = outputPath debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir) if err != nil { - return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err) + return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err) } inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{ Resolved: resolved, @@ -275,11 +275,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, Promptkit: req.Config.Promptkit, }) if err != nil { - return nil, err + return result, err } + result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName collection, err := collectWeather(ctx, req.Config, req.Collector) if err != nil { - return nil, err + return result, err } return generatePromptReport(ctx, promptReportRequest{ GenerateRequest: req, @@ -287,6 +288,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult, Collection: *collection, Inspection: inspection, DebugWriter: debugWriter, + Result: result, }) } @@ -339,14 +341,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro return nil, err } if req.Batch == BatchEvening || req.Batch == BatchMorning { - store := req.Store - if store == nil { - defaultStore, err := defaultStore(req.Config) - if err != nil { - return nil, err - } - store = defaultStore - } startedAt := now result := &BatchResult{Batch: req.Batch, StartedAt: startedAt} for _, planned := range plannedReports { @@ -362,7 +356,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro OutputPath: outputPath, Notifier: req.Notifier, Executor: req.Executor, - Store: store, }, Resolved: resolved, Collection: *collection, @@ -398,13 +391,14 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro } func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) { - item.DataPackagePath = result.DataPackagePath - item.PreparationPath = result.PreparationPath - item.ExecutionPath = result.ExecutionPath item.LLMDebugPath = result.LLMDebugPath - item.ReportPath = result.ReportPath item.OutputPath = result.OutputPath - item.MetadataPath = result.MetadataPath + item.ProfileID = result.ProfileID + item.BackendID = result.BackendID + item.ModelName = result.ModelName + item.Timezone = result.Timezone + item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...) + item.ValidationStatus = result.ValidationStatus if result.Notification != nil { item.NotificationStatus = result.Notification.Status item.NotificationRunID = result.Notification.RunID @@ -451,6 +445,7 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult { RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, ValidPeriod: metadata.ValidPeriod, + Timezone: "", } } @@ -612,78 +607,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda return bundle, nil } -type finalizeRenderedReportRequest struct { - Config config.Config - Store state.Store - Resolved report.Resolved - Metadata state.Metadata - MetadataPath string - ExecutionArtifact *state.PromptExecutionArtifact - RenderedReportPath string - OutputPath string - Notifier Notifier - GenerationErr error - noNotify bool -} - -type finalizeRenderedReportResult struct { - OutputPath string - Metadata state.Metadata - MetadataPath string - Notification *NotificationResult -} - -func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) { - if req.Store == nil { - return finalizeRenderedReportResult{}, fmt.Errorf("state store is required") - } - if req.RenderedReportPath == "" { - return finalizeRenderedReportResult{}, fmt.Errorf("rendered report path is required for report %q", req.Resolved.Definition.ID) - } - if req.ExecutionArtifact == nil { - return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID) - } - - result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath} - if req.OutputPath != "" && req.GenerationErr == nil { - if req.OutputPath != req.RenderedReportPath { - if err := fileutil.CopyFileAtomic(req.RenderedReportPath, req.OutputPath); err != nil { - return result, err - } - result.OutputPath = req.OutputPath - if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) { - paths.OutputPath = req.OutputPath - }); err != nil { - return result, err - } - } else { - result.OutputPath = req.OutputPath - } - } - - metadata := req.Metadata - metadata.RenderedReportPath = req.RenderedReportPath - metadataPath, err := req.Store.SaveMetadata(ctx, metadata) - if err != nil { - return result, err - } - result.Metadata = metadata - result.MetadataPath = metadataPath - if req.GenerationErr != nil { - return result, req.GenerationErr - } - if req.noNotify { - return result, nil - } - - notification, err := notifyReport(ctx, req.Config, req.Resolved, req.OutputPath, metadata.RunID, metadata.GeneratedAt, req.Notifier) - result.Notification = notification - if err != nil { - return result, err - } - return result, nil -} - func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) { notifier, enabled := reportNotifier(cfg, notifier) if !enabled { @@ -965,7 +888,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected } } -func promptMetadata(metadata state.Metadata) promptinput.Metadata { +func promptMetadata(metadata briefing.Metadata) promptinput.Metadata { return promptinput.Metadata{ RunID: metadata.RunID, ReportID: metadata.ReportID, @@ -1008,10 +931,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext { return &location } -func defaultStore(cfg config.Config) (*state.FilesystemStore, error) { - return state.NewFilesystemStore(cfg.Workspace) -} - func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error { if err == nil { return nil diff --git a/internal/app/batch_execution_test.go b/internal/app/batch_execution_test.go deleted file mode 100644 index 39e06c0..0000000 --- a/internal/app/batch_execution_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package app - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" - "gitea.maximumdirect.net/eric/weatherreporter/internal/config" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" -) - -func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) { - tests := []struct { - name string - batch BatchKind - now string - wantPrompts int - }{ - {name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3}, - {name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := config.Defaults() - cfg.Workspace.Root = t.TempDir() - now := mustParse(test.now) - req := BatchRequest{Config: cfg, Batch: test.batch, Now: now, WorkingDir: t.TempDir()} - candidates, err := batchInspectionCandidates(req, now) - if err != nil { - t.Fatalf("batchInspectionCandidates() error = %v", err) - } - executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{ - "weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"}, - }, prompts: map[string]promptexec.PromptInspection{}} - for _, candidate := range candidates { - executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition) - } - collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) { - return nil, errors.New("collection reached") - }) - req.Executor = executor - req.Collector = collector - _, err = RunBatchDetailed(context.Background(), req) - if err == nil || err.Error() != "collection reached" { - t.Fatalf("RunBatchDetailed() error = %v, want collection error", err) - } - if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" { - t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests) - } - }) - } -} - -func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) { - item := BatchReportResult{} - copyBatchReportPaths(&item, &ReportResult{ - DataPackagePath: "/runs/daily/data_package.yaml", - PreparationPath: "/runs/daily/preparation.json", - }) - - data, err := json.Marshal(item) - if err != nil { - t.Fatalf("Marshal() error = %v", err) - } - text := string(data) - for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} { - if strings.Contains(text, omitted) { - t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text) - } - } - if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") { - t.Fatalf("batch item omits reached paths:\n%s", text) - } -} - -type collectorFunc func(context.Context, collect.Request) (*collect.Result, error) - -func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) { - return f(ctx, req) -} - -var _ Collector = collectorFunc(nil) diff --git a/internal/app/batch_workflow_test.go b/internal/app/batch_workflow_test.go deleted file mode 100644 index e7351b3..0000000 --- a/internal/app/batch_workflow_test.go +++ /dev/null @@ -1,449 +0,0 @@ -package app - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" - "gitea.maximumdirect.net/eric/weatherreporter/internal/config" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" - "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" -) - -type assembledBatchExecutor struct { - definitions map[string]report.Definition - promptRequests []string - profileRequests []string - executeRequests []promptexec.ExecuteRequest - failures map[int]error - active int - maxActive int -} - -func newAssembledBatchExecutor() *assembledBatchExecutor { - definitions := make(map[string]report.Definition) - for _, definition := range report.DefaultRegistry().All() { - definitions[definition.PromptID] = definition - } - return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)} -} - -func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { - e.promptRequests = append(e.promptRequests, id+"@"+version) - definition, ok := e.definitions[id] - if !ok || definition.PromptVersion != version { - return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection") - } - return logicalPromptInspection(definition), nil -} - -func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { - e.profileRequests = append(e.profileRequests, id) - return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil -} - -func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { - call := len(e.executeRequests) - e.executeRequests = append(e.executeRequests, req) - e.active++ - if e.active > e.maxActive { - e.maxActive = e.active - } - defer func() { e.active-- }() - stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - preparation := promptexec.Preparation{ - PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", - RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", - ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, - } - if err := callback(preparation, nil); err != nil { - return nil, err - } - if err := e.failures[call]; err != nil { - return nil, err - } - definition := e.definitions[req.PromptID] - return &promptexec.Execution{ - RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, - PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, - BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash", - StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, - RawOutput: []byte(generatedTextForPrompt(req.PromptID)), - Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil), - }, nil -} - -type assembledBatchNotifier struct { - reportRequests []NotificationRequest - batchRequests []batchNotificationRequest - batchResult *NotificationResult - batchErr error -} - -func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) { - n.reportRequests = append(n.reportRequests, req) - return nil, errors.New("per-report notification must be suppressed") -} - -func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) { - n.batchRequests = append(n.batchRequests, req) - if n.batchErr != nil { - return nil, n.batchErr - } - if n.batchResult != nil { - result := *n.batchResult - if result.PipelineID == "" { - result.PipelineID = req.PipelineID - } - if result.BundleID == "" { - result.BundleID = req.BundleID - } - if result.IdempotencyKey == "" { - result.IdempotencyKey = req.IdempotencyKey - } - return &result, nil - } - return &NotificationResult{ - RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID, - IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted", - }, nil -} - -func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) { - tests := []struct { - name string - batch BatchKind - now time.Time - wantIDs []report.ID - wantCopies []string - }{ - {name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}}, - {name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := assembledBatchConfig(t, false) - bundle := assembledBatchBundle(t, "2026-05-31") - collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} - executor := newAssembledBatchExecutor() - outputDir := filepath.Join(t.TempDir(), "output") - debugRoot := filepath.Join(t.TempDir(), "debug") - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: test.batch, Now: test.now, WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot, - Collector: collector, Executor: executor, - }) - if err != nil { - t.Fatalf("RunBatchDetailed() error = %v", err) - } - if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 { - t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed) - } - if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 { - t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs)) - } - if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" { - t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests) - } - for index, item := range result.Reports { - if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" { - t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index]) - } - if executor.executeRequests[index].PromptID != item.PromptID { - t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID) - } - if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" { - t.Fatalf("successful report paths = %#v", item) - } - assertBatchItemMatchesMetadata(t, item) - if filepath.Base(item.OutputPath) != test.wantCopies[index] { - t.Fatalf("output = %q, want %q", item.OutputPath, test.wantCopies[index]) - } - assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath) - renderedReport, readErr := os.ReadFile(item.ReportPath) - if readErr != nil { - t.Fatalf("read rendered report: %v", readErr) - } - copied, readErr := os.ReadFile(item.OutputPath) - if readErr != nil || !bytes.Equal(renderedReport, copied) { - t.Fatalf("output mismatch/error = %v", readErr) - } - } - }) - } -} - -func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) { - cfg := assembledBatchConfig(t, true) - bundle := assembledBatchBundle(t, "2026-05-31") - collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} - executor := newAssembledBatchExecutor() - executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil) - notifier := &assembledBatchNotifier{} - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), - OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"), - Collector: collector, Executor: executor, Notifier: notifier, - }) - if err != nil { - t.Fatalf("RunBatchDetailed() error = %v", err) - } - if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 { - t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed) - } - if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" { - t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification) - } - for index, item := range result.Reports { - if index == 1 { - if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) { - t.Fatalf("failed item = %#v", item) - } - if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" { - t.Fatalf("failed item reached paths = %#v", item) - } - assertBatchItemMatchesMetadata(t, item) - continue - } - if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" { - t.Fatalf("continued item %d = %#v", index, item) - } - } -} - -func TestRunBatchDetailedNotificationLifecycle(t *testing.T) { - t.Run("disabled", func(t *testing.T) { - cfg := assembledBatchConfig(t, false) - bundle := assembledBatchBundle(t, "2026-05-31") - notifier := &assembledBatchNotifier{} - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, - }) - if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 { - t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests)) - } - }) - - t.Run("batch disabled", func(t *testing.T) { - cfg := assembledBatchConfig(t, true) - cfg.Notify.Distributor.Batch.Enabled = false - bundle := assembledBatchBundle(t, "2026-05-31") - notifier := &assembledBatchNotifier{} - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, - }) - if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 { - t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests)) - } - }) - - t.Run("all success", func(t *testing.T) { - cfg := assembledBatchConfig(t, true) - bundle := assembledBatchBundle(t, "2026-05-31") - notifier := &assembledBatchNotifier{batchResult: &NotificationResult{ - RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted", - Report: []byte(`{"actions":[{"action":"replace_older"}]}`), - }} - outputDir := filepath.Join(t.TempDir(), "output") - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, - }) - if err != nil { - t.Fatalf("RunBatchDetailed() error = %v", err) - } - if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 { - t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests)) - } - outputPaths := make(map[string]struct{}, len(result.Reports)) - for _, item := range result.Reports { - outputPaths[item.OutputPath] = struct{}{} - } - request := notifier.batchRequests[0] - if len(request.IncludedReports) != len(result.Reports) { - t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports)) - } - for _, file := range request.Files { - if _, ok := outputPaths[file.SourcePath]; !ok || !strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" { - t.Fatalf("notification file = %#v, want selected Markdown output source", file) - } - } - }) - - t.Run("upload failure", func(t *testing.T) { - cfg := assembledBatchConfig(t, true) - bundle := assembledBatchBundle(t, "2026-05-31") - notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")} - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, - }) - if err != nil { - t.Fatalf("RunBatchDetailed() error = %v", err) - } - if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" { - t.Fatalf("result = %#v, want successful reports and failed notification", result) - } - for _, item := range result.Reports { - if item.Status != "succeeded" { - t.Fatalf("report item = %#v, want success despite notification failure", item) - } - } - if !strings.Contains(result.Notification.Error, "batch upload rejected") { - t.Fatalf("notification error = %#v", result.Notification) - } - }) - - t.Run("status report", func(t *testing.T) { - cfg := assembledBatchConfig(t, true) - bundle := assembledBatchBundle(t, "2026-05-31") - notifier := &assembledBatchNotifier{batchResult: &NotificationResult{ - RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted", - StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`), - }} - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier, - }) - if err != nil || result.Failed != 0 || result.Notification == nil { - t.Fatalf("result/error = %#v/%v", result, err) - } - if result.Notification.Status != "accepted" || result.Notification.RunID != "batch-notification-run" { - t.Fatalf("notification = %#v", result.Notification) - } - }) -} - -func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) { - cfg := assembledBatchConfig(t, false) - bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01") - outputDir := filepath.Join(t.TempDir(), "output") - debugRoot := filepath.Join(t.TempDir(), "debug") - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot, - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), - }) - if err != nil || result.Failed != 0 || len(result.Reports) != 3 { - t.Fatalf("result/error = %#v/%v", result, err) - } - seenRuns := make(map[string]struct{}) - seenDebug := make(map[string]struct{}) - dailyDates := make(map[string]BatchReportResult) - for _, item := range result.Reports { - if _, exists := seenRuns[item.RunID]; exists { - t.Fatalf("duplicate run ID %q", item.RunID) - } - seenRuns[item.RunID] = struct{}{} - if _, exists := seenDebug[item.LLMDebugPath]; exists { - t.Fatalf("duplicate debug path %q", item.LLMDebugPath) - } - seenDebug[item.LLMDebugPath] = struct{}{} - if item.ReportID == report.Daily { - date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02") - dailyDates[date] = item - } - } - for _, date := range []string{"2026-05-31", "2026-06-01"} { - item, ok := dailyDates[date] - if !ok { - t.Fatalf("daily items = %#v, want %s", dailyDates, date) - } - if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") { - t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath) - } - wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID) - if item.LLMDebugPath != wantDebugPrefix { - t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix) - } - assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json")) - } -} - -func assembledBatchConfig(t *testing.T, notify bool) config.Config { - t.Helper() - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = notify - cfg.Notify.Distributor.Batch.Enabled = notify - return cfg -} - -func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle { - t.Helper() - bundle := workflowBundle(t) - location := mustLoadTestLocation(t, "America/Chicago") - for _, date := range dates { - periods := fullDayPeriods(t, date, location) - for index := range periods { - temperature := float64(60 + index) - periods[index].TemperatureF = &temperature - periods[index].TextDescription = "Partly cloudy" - } - bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...) - } - return bundle -} - -func generatedTextForPrompt(promptID string) string { - switch promptID { - case "weather.today_generated_text": - return validTodayWorkflowJSON() - case "weather.tomorrow_generated_text": - return validTomorrowWorkflowJSON() - case "weather.daily_generated_text": - return validDailyWorkflowJSON() - case "weather.hourly_generated_text": - return validHourlyWorkflowJSON() - default: - return "" - } -} - -func assertBatchPathsExist(t *testing.T, paths ...string) { - t.Helper() - for _, path := range paths { - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected path %q: %v", path, err) - } - } -} - -func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) { - t.Helper() - data, err := os.ReadFile(item.MetadataPath) - if err != nil { - t.Fatalf("read metadata %q: %v", item.MetadataPath, err) - } - var metadata state.Metadata - if err := json.Unmarshal(data, &metadata); err != nil { - t.Fatalf("decode metadata %q: %v", item.MetadataPath, err) - } - if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID || - item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath || - item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath { - t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata) - } -} - -func loadBatchDataPackage(t *testing.T, path string) promptinput.Package { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read data package: %v", err) - } - pkg, err := promptinput.LoadYAML(data) - if err != nil { - t.Fatalf("LoadYAML() error = %v", err) - } - return pkg -} diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go new file mode 100644 index 0000000..680376f --- /dev/null +++ b/internal/app/generation_test.go @@ -0,0 +1,103 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" +) + +type generationCollector struct { + bundle *weatherdata.Bundle + err error +} + +func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) { + return &collect.Result{Bundle: c.bundle}, c.err +} + +type generationExecutor struct{ called bool } + +func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { + definition := report.DefaultRegistry().MustLookup(report.Daily) + return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil +} +func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { + return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil +} +func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { + stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil { + return nil, err + } + e.called = true + return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil +} + +func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) { + cfg := config.Defaults() + cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home" + bundle := generationBundle(t) + executor := &generationExecutor{} + workingDir := t.TempDir() + result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor}) + if err != nil { + t.Fatalf("GenerateDetailed() error = %v", err) + } + if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" { + t.Fatalf("result = %#v", result) + } + if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) { + t.Fatalf("workspace state = %v, want absent", err) + } + data, err := os.ReadFile(result.OutputPath) + if err != nil || len(data) == 0 { + t.Fatalf("output = %q, error = %v", data, err) + } +} + +func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) { + cfg := config.Defaults() + cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home" + collectionErr := errors.New("weather source unavailable") + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), + WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{}, + }) + if !errors.Is(err, collectionErr) { + t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr) + } + if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" { + t.Fatalf("result = %#v", result) + } +} + +func generationBundle(t *testing.T) weatherdata.Bundle { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) + if err != nil { + t.Fatalf("read bundle fixture: %v", err) + } + var bundle weatherdata.Bundle + if err := json.Unmarshal(data, &bundle); err != nil { + t.Fatalf("decode bundle fixture: %v", err) + } + return bundle +} +func generationTime(value string) time.Time { + parsed, _ := time.Parse(time.RFC3339, value) + return parsed +} + +var _ promptexec.Executor = (*generationExecutor)(nil) +var _ Collector = generationCollector{} +var _ = report.Daily diff --git a/internal/app/inspect.go b/internal/app/inspect.go index aa5b645..cd25f5d 100644 --- a/internal/app/inspect.go +++ b/internal/app/inspect.go @@ -33,7 +33,7 @@ type SourceInspection struct { } func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) { - store, err := defaultStore(req.Config) + store, err := state.NewFilesystemStore(req.Config.Workspace) if err != nil { return nil, err } @@ -94,7 +94,7 @@ type runInspection struct { } func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) { - store, err := defaultStore(req.Config) + store, err := state.NewFilesystemStore(req.Config.Workspace) if err != nil { return runInspection{}, err } diff --git a/internal/app/output_test.go b/internal/app/output_test.go deleted file mode 100644 index f1a86e9..0000000 --- a/internal/app/output_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package app - -import ( - "context" - "errors" - "os" - "path/filepath" - "testing" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" - "gitea.maximumdirect.net/eric/weatherreporter/internal/config" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" -) - -func TestResolveReportOutputPath(t *testing.T) { - workingDir := t.TempDir() - cfg := config.Defaults() - now := workflowTime("2026-05-29T08:30:00-05:00") - - for _, test := range []struct { - report ReportKind - date string - name string - }{ - {report: ReportDaily, date: "2026-05-30T12:00:00-05:00", name: "daily-2026-05-30.md"}, - {report: ReportToday, date: "2026-05-29T12:00:00-05:00", name: "today.md"}, - {report: ReportTomorrow, name: "tomorrow.md"}, - {report: ReportHourly, name: "hourly.md"}, - } { - t.Run(string(test.report), func(t *testing.T) { - req := GenerateRequest{Config: cfg, Report: test.report} - if test.date != "" { - req.Date = workflowTime(test.date) - } - resolved, err := ResolveGenerate(req, now) - if err != nil { - t.Fatalf("ResolveGenerate() error = %v", err) - } - path, err := resolveReportOutputPath(workingDir, "", resolved) - if err != nil { - t.Fatalf("resolveReportOutputPath() error = %v", err) - } - if path != filepath.Join(workingDir, test.name) { - t.Fatalf("path = %q, want %q", path, filepath.Join(workingDir, test.name)) - } - }) - } - - resolved, err := ResolveGenerate(GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-30T12:00:00-05:00"), - }, now) - if err != nil { - t.Fatalf("ResolveGenerate() error = %v", err) - } - absoluteDir := t.TempDir() - for _, test := range []struct { - override string - want string - }{ - {override: filepath.Join("reports", "custom.md"), want: filepath.Join(workingDir, "reports", "custom.md")}, - {override: filepath.Join(absoluteDir, "custom.md"), want: filepath.Join(absoluteDir, "custom.md")}, - } { - path, err := resolveReportOutputPath(workingDir, test.override, resolved) - if err != nil { - t.Fatalf("resolveReportOutputPath(%q) error = %v", test.override, err) - } - if path != filepath.Clean(test.want) { - t.Fatalf("path = %q, want %q", path, filepath.Clean(test.want)) - } - } -} - -func TestGenerateDetailedRejectsInvalidOutputBeforeCollection(t *testing.T) { - cfg := workflowConfig(t) - collector := &workflowCollector{err: errors.New("collection must not run")} - _, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), - Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: t.TempDir(), - Collector: collector, - }) - if err == nil || collector.calls != 0 { - t.Fatalf("GenerateDetailed() error/calls = %v/%d, want invalid output before collection", err, collector.calls) - } -} - -func TestGenerateDetailedPreservesExistingOutputWhenGenerationFails(t *testing.T) { - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = false - workingDir := t.TempDir() - outputPath := filepath.Join(workingDir, "daily-2026-05-29.md") - if err := os.WriteFile(outputPath, []byte("existing report"), 0o600); err != nil { - t.Fatalf("write existing output: %v", err) - } - bundle := workflowBundle(t) - result, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), - Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, - Executor: &workflowExecutor{definition: report.DefaultRegistry().MustLookup(report.Daily), raw: []byte(`{}`)}, - }) - if err == nil || result == nil || result.OutputPath != "" { - t.Fatalf("result/error/output = %#v/%v/%q, want failed generation without output publication", result, err, result.OutputPath) - } - data, readErr := os.ReadFile(outputPath) - if readErr != nil || string(data) != "existing report" { - t.Fatalf("output after failure = %q, error %v, want preserved content", data, readErr) - } -} - -func TestRunBatchDetailedUsesWorkingDirectoryForOutput(t *testing.T) { - cfg := assembledBatchConfig(t, false) - workingDir := t.TempDir() - bundle := assembledBatchBundle(t, "2026-05-31") - result, err := RunBatchDetailed(context.Background(), BatchRequest{ - Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: workingDir, - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, - Executor: newAssembledBatchExecutor(), - }) - if err != nil || result == nil || result.Failed != 0 { - t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err) - } - if len(result.Reports) != 2 { - t.Fatalf("reports = %#v, want Tomorrow and Daily", result.Reports) - } - if result.Reports[0].OutputPath != filepath.Join(workingDir, "tomorrow.md") || - result.Reports[1].OutputPath != filepath.Join(workingDir, "daily-2026-05-31.md") { - t.Fatalf("output paths = %#v, want working-directory defaults", result.Reports) - } -} diff --git a/internal/app/prompt_artifact_paths_test.go b/internal/app/prompt_artifact_paths_test.go deleted file mode 100644 index fdbb324..0000000 --- a/internal/app/prompt_artifact_paths_test.go +++ /dev/null @@ -1,523 +0,0 @@ -package app - -import ( - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" - "gitea.maximumdirect.net/eric/weatherreporter/internal/config" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" - "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" -) - -const ( - failPromptExecution = "prompt execution" - failMetadata = "metadata" - failGeneratedText = "generated text" - failRenderContext = "render context" - failRenderedReportPath = "rendered report path" -) - -type failingPersistenceStore struct { - state.Store - failOperation string - failExecutionCall int - failMetadataCall int - executionCalls int - metadataCalls int - renderedReportPath string -} - -func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) { - s.executionCalls++ - if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) { - return "", errors.New("injected prompt execution persistence failure") - } - return s.Store.SavePromptExecution(ctx, resolved, artifact) -} - -func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) { - if s.failOperation == failGeneratedText { - return "", errors.New("injected generated text persistence failure") - } - return s.Store.SaveGeneratedText(ctx, resolved, data) -} - -func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) { - if s.failOperation == failRenderContext { - return "", errors.New("injected render context persistence failure") - } - return s.Store.SaveRenderContext(ctx, resolved, value) -} - -func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) { - if s.failOperation == failRenderedReportPath { - return s.renderedReportPath, nil - } - return s.Store.PrepareRenderedReport(ctx, resolved) -} - -func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) { - s.metadataCalls++ - if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall { - return "", errors.New("injected metadata persistence failure") - } - return s.Store.SaveMetadata(ctx, metadata) -} - -type artifactPathExecutor struct { - beforePreparationErr error - afterPreparationErr error - validation promptexec.ValidationStatus -} - -func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) { - return promptexec.PromptInspection{}, errors.New("unexpected inspection") -} - -func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) { - return promptexec.ProfileInspection{}, errors.New("unexpected inspection") -} - -func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { - if e.beforePreparationErr != nil { - return nil, e.beforePreparationErr - } - now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - if err := callback(promptexec.Preparation{ - PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", - RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test", - ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now, - }, nil); err != nil { - return nil, err - } - if e.afterPreparationErr != nil { - return nil, e.afterPreparationErr - } - validation := e.validation - if validation == "" { - validation = promptexec.ValidationPassed - } - return &promptexec.Execution{ - RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, - PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, - BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash", - StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, - RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), - Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil), - }, nil -} - -type successfulNotifier struct{} - -func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { - return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil -} - -type failingNotifier struct{} - -func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { - return nil, errors.New("injected notification failure") -} - -func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { - tests := []struct { - name string - failOperation string - failMetadataCall int - output bool - notify bool - want reachedPromptArtifacts - }{ - {name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}}, - {name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}}, - {name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}}, - {name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}}, - {name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}}, - {name: "rendered report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}}, - {name: "output then metadata", failOperation: failMetadata, failMetadataCall: 5, output: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - req, paths := promptArtifactRequest(t, artifactPathExecutor{}) - store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall} - req.Store = store - if test.output { - req.OutputPath = filepath.Join(t.TempDir(), "daily.md") - paths.output = req.OutputPath - } - if test.notify { - req.Config.Notify.Distributor.Enabled = true - req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" - req.Notifier = successfulNotifier{} - req.noNotify = false - } - - result, err := generatePromptReport(context.Background(), req) - if err == nil || result == nil { - t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) - } - assertReachedPromptArtifacts(t, result, paths, test.want) - }) - } -} - -func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) { - tests := []struct { - name string - executor artifactPathExecutor - want reachedPromptArtifacts - wantExecutionStatus state.PromptExecutionStatus - wantExecutionPaths state.PromptExecutionPaths - wantRawExecution bool - }{ - { - name: "preparation failure", - executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)}, - want: reachedPromptArtifacts{preparation: true, metadata: true}, - }, - { - name: "operational execution failure", - executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)}, - want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true}, - wantExecutionStatus: state.PromptExecutionFailed, - }, - { - name: "completed validation rejection", - executor: artifactPathExecutor{validation: promptexec.ValidationFailed}, - want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, - wantExecutionStatus: state.PromptExecutionValidationRejected, - wantRawExecution: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - req, paths := promptArtifactRequest(t, test.executor) - if test.wantRawExecution { - test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw - } - result, err := generatePromptReport(context.Background(), req) - if err == nil || result == nil { - t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) - } - assertReachedPromptArtifacts(t, result, paths, test.want) - if test.wantExecutionStatus != "" { - artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath) - if loadErr != nil { - t.Fatalf("LoadPromptExecution() error = %v", loadErr) - } - if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths { - t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths) - } - } - }) - } -} - -func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) { - req, paths := promptArtifactRequest(t, artifactPathExecutor{}) - req.OutputPath = filepath.Join(t.TempDir(), "daily.md") - paths.output = req.OutputPath - req.Config.Notify.Distributor.Enabled = true - req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" - req.Notifier = successfulNotifier{} - req.noNotify = false - - result, err := generatePromptReport(context.Background(), req) - if err != nil { - t.Fatalf("generatePromptReport() error = %v", err) - } - want := state.PromptExecutionPaths{ - RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText, - RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport, - OutputPath: paths.output, - } - assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want) - - data, err := os.ReadFile(result.ExecutionPath) - if err != nil { - t.Fatalf("read execution artifact: %v", err) - } - text := string(data) - for _, forbidden := range []string{ - "Showers are possible during the selected day", `"rawOutput":`, `"debug":`, - `"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`, - "credential", "secret-value", - } { - if strings.Contains(text, forbidden) { - t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text) - } - } -} - -func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) { - tests := []struct { - name string - failOperation string - failExecutionCall int - failMetadataCall int - requestOutput bool - failOutput bool - notify bool - notificationFailure bool - wantExecution reachedExecutionArtifacts - wantResult reachedPromptArtifacts - }{ - { - name: "normalized text write", failOperation: failGeneratedText, - wantExecution: reachedExecutionArtifacts{raw: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, - }, - { - name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2, - wantExecution: reachedExecutionArtifacts{raw: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, - }, - { - name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, - }, - { - name: "render context write", failOperation: failRenderContext, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, - }, - { - name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, - }, - { - name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, - }, - { - name: "rendered report write", failOperation: failRenderedReportPath, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, - }, - { - name: "rendered report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, - }, - { - name: "output write", requestOutput: true, failOutput: true, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, - }, - { - name: "output checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, - }, - { - name: "output metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, - }, - { - name: "notification operation", requestOutput: true, notify: true, notificationFailure: true, - wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, - wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - req, paths := promptArtifactRequest(t, artifactPathExecutor{}) - store := &failingPersistenceStore{ - Store: req.Store, failOperation: test.failOperation, - failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall, - } - if test.failOperation == failRenderedReportPath { - store.renderedReportPath = t.TempDir() - } - req.Store = store - if test.requestOutput { - req.OutputPath = filepath.Join(t.TempDir(), "daily.md") - paths.output = req.OutputPath - } - if test.failOutput { - blocker := filepath.Join(t.TempDir(), "not-a-directory") - if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil { - t.Fatalf("write output blocker: %v", err) - } - req.OutputPath = filepath.Join(blocker, "daily.md") - paths.output = req.OutputPath - } - if test.notify { - req.Config.Notify.Distributor.Enabled = true - req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" - req.Notifier = successfulNotifier{} - req.noNotify = false - } - if test.notificationFailure { - req.Notifier = failingNotifier{} - } - - result, err := generatePromptReport(context.Background(), req) - if err == nil || result == nil { - t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) - } - assertReachedPromptArtifacts(t, result, paths, test.wantResult) - assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution)) - }) - } -} - -type reachedExecutionArtifacts struct { - raw bool - normalized bool - renderContext bool - report bool - output bool -} - -func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths { - result := state.PromptExecutionPaths{} - if reached.raw { - result.RawOutputPath = paths.GeneratedTextRaw - } - if reached.normalized { - result.GeneratedTextPath = paths.GeneratedText - } - if reached.renderContext { - result.RenderContextPath = paths.RenderContext - } - if reached.report { - result.RenderedReportPath = paths.RenderedReport - } - if reached.output { - result.OutputPath = paths.output - } - return result -} - -func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) { - t.Helper() - artifact, err := store.LoadPromptExecution(context.Background(), path) - if err != nil { - t.Fatalf("LoadPromptExecution() error = %v", err) - } - if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed { - t.Fatalf("execution outcome changed after downstream write: %#v", artifact) - } - if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" { - t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance) - } - if artifact.Paths != want { - t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want) - } -} - -type promptArtifactPaths struct { - state.ArtifactPaths - output string -} - -type reachedPromptArtifacts struct { - preparation bool - execution bool - metadata bool - raw bool - normalized bool - renderContext bool - report bool - output bool -} - -func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) { - t.Helper() - cfg := config.Defaults() - cfg.Workspace.Root = t.TempDir() - resolved, err := ResolveGenerate(GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"), - }, mustParse("2026-05-29T05:00:00-05:00")) - if err != nil { - t.Fatalf("ResolveGenerate() error = %v", err) - } - bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) - if err != nil { - t.Fatalf("read daily fixture: %v", err) - } - var bundle weatherdata.Bundle - if err := json.Unmarshal(bundleData, &bundle); err != nil { - t.Fatalf("decode daily fixture: %v", err) - } - filesystemStore, err := state.NewFilesystemStore(cfg.Workspace) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - paths, err := filesystemStore.Paths(resolved) - if err != nil { - t.Fatalf("Paths() error = %v", err) - } - debugWriter, err := promptdebug.NewPromptDebugWriter("") - if err != nil { - t.Fatalf("NewPromptDebugWriter() error = %v", err) - } - return promptReportRequest{ - GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore}, - Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, - Inspection: PromptInspectionResult{ - PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, - PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model", - }, - DebugWriter: debugWriter, noNotify: true, - }, promptArtifactPaths{ArtifactPaths: paths} -} - -func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) { - t.Helper() - if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage { - t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage) - } - if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata { - t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata) - } - checks := []struct { - name string - got string - metadataGot string - inMetadata bool - path string - want bool - }{ - {"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation}, - {"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution}, - {"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata}, - {"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw}, - {"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized}, - {"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext}, - {"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report}, - {"output", result.OutputPath, "", false, paths.output, want.output}, - } - for _, check := range checks { - if check.want && check.got != check.path { - t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path) - } - if check.want && check.inMetadata && check.metadataGot != check.path { - t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path) - } - if !check.want && check.got != "" { - t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got) - } - if !check.want && check.inMetadata && check.metadataGot != "" { - t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot) - } - } -} diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index 3fc2e15..71d909c 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -3,7 +3,6 @@ package app import ( "context" "fmt" - "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" @@ -14,7 +13,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) type promptReportRequest struct { @@ -23,9 +22,23 @@ type promptReportRequest struct { Collection collect.Result Inspection PromptInspectionResult DebugWriter *promptdebug.PromptDebugWriter + Result *ReportResult noNotify bool } +type promptReportWorkflow struct { + ctx context.Context + req promptReportRequest + result *ReportResult + briefingMetadata briefing.Metadata + reportFacts ReportFacts + moduleSnapshot module.Snapshot + dataPackage []byte + handler generatedtext.Handler + debugRef promptdebug.PromptDebugRef + callbackFailed bool +} + func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { workflow, err := newPromptReportWorkflow(ctx, req) if err != nil { @@ -34,369 +47,145 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report if err := workflow.buildInputs(); err != nil { return workflow.result, err } - - execution, executeErr := workflow.executePrompt() - if executeErr != nil { - return workflow.result, workflow.handleExecutionFailure(executeErr) + execution, err := workflow.executePrompt() + if err != nil { + if workflow.callbackFailed { + return workflow.result, err + } + return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err)) } if execution == nil { - err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil) - if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil { - return workflow.result, saveErr - } - return workflow.result, workflow.reportError("execute prompt", err) + return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)) } - if err := workflow.persistExecutionDebug(*execution); err != nil { + workflow.result.ValidationStatus = execution.Validation.Status + if err := workflow.writeExecutionDebug(*execution); err != nil { return workflow.result, err } - if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { - err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil) - if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil { - return workflow.result, saveErr - } - return workflow.result, workflow.reportError("validate prompt execution", err) - } - if err := workflow.persistCompletedExecution(*execution); err != nil { - return workflow.result, err + return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)) } if execution.Validation.Status == promptexec.ValidationFailed { - err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil) - return workflow.result, workflow.reportError("validate prompt execution", err) + return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)) } - - rendered, err := workflow.persistGeneratedContent(execution.RawOutput) - if err != nil { - return workflow.result, err - } - return workflow.finalizeReport(rendered) -} - -type promptReportWorkflow struct { - ctx context.Context - req promptReportRequest - store state.Store - result *ReportResult - metadata state.Metadata - briefingMetadata briefing.Metadata - reportFacts ReportFacts - moduleSnapshot module.Snapshot - dataPackageBytes []byte - handler generatedtext.Handler - executionArtifact state.PromptExecutionArtifact - debugRef promptdebug.PromptDebugRef - prepared bool - callbackFailed bool + return workflow.renderAndPublish(execution.RawOutput) } func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) { if req.Collection.Bundle == nil { return nil, fmt.Errorf("collected weather bundle is required") } - store := req.Store - var err error - if store == nil { - store, err = defaultStore(req.Config) - if err != nil { - return nil, err - } + result := req.Result + if result == nil { + result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection) + } + return &promptReportWorkflow{ + ctx: ctx, req: req, + result: result, + }, nil +} + +func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult { + metadata := resolved.Metadata() + return &ReportResult{ + ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name, + PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, + RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone, + ValidPeriod: metadata.ValidPeriod, + ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName, } - return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil } func (w *promptReportWorkflow) buildInputs() error { - paths, err := w.store.Paths(w.req.Resolved) - if err != nil { - return err - } - w.result = &ReportResult{} + var err error w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle) if err != nil { - return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err) + return w.reportError("build report facts", err) } w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts) if err != nil { - return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err) + return w.reportError("build module snapshot", err) } - moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot) - if err != nil { - return err - } - w.result.ModuleSnapshot = w.moduleSnapshot - w.result.ModuleSnapshotPath = moduleSnapshotPath w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected)) - w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{ - ModuleSnapshot: moduleSnapshotPath, - Metadata: paths.Metadata, - }) - w.result.Metadata = w.metadata - dataPackage, err := promptinput.Build(promptinput.BuildRequest{ - Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot, - }) + w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...) + dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot}) if err != nil { return w.reportError("build data package", err) } - w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage) + w.dataPackage, err = promptinput.MarshalYAML(dataPackage) if err != nil { - return err + return w.reportError("marshal data package", err) } - dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes) - if err != nil { - return err - } - w.metadata.DataPackagePath = dataPackagePath - w.result.DataPackage = dataPackage - w.result.DataPackagePath = dataPackagePath - w.result.Metadata = w.metadata w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition) if err != nil { return w.reportError("lookup generated text catalog", err) } - w.debugRef = promptdebug.PromptDebugRef{ - ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID, - } + w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID} return nil } func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) { - return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{ - PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, - ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes, - DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(), - }, w.persistPreparation) + captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled() + return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug) } -func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { - artifact := state.PromptPreparationArtifact{ - SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded, - ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID, - PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, - DataPackagePath: w.result.DataPackagePath, Preparation: &preparation, - StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration, +func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { + w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName + if w.req.DebugWriter == nil { + return nil } - path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact) - if err != nil { - w.callbackFailed = true - return err - } - w.prepared = true - w.result.PreparationPath = path - w.metadata.PreparationPath = path - w.result.Metadata = w.metadata - debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug) + path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug) if err != nil { w.callbackFailed = true return promptDebugWriteError(err) } - if debugPath != "" { - w.result.LLMDebugPath = debugPath - } - if err := w.saveMetadata(); err != nil { - w.callbackFailed = true - return err - } + w.result.LLMDebugPath = path return nil } -func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error { - if w.callbackFailed { - return executeErr +func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error { + if w.req.DebugWriter == nil { + return nil } - executeErr = classifiedPromptError("prompt execution failed", executeErr) - if !w.prepared { - if err := w.persistPreparationFailure(executeErr); err != nil { - return err - } - return w.reportError("prepare prompt", executeErr) - } - if promptexec.CategoryOf(executeErr) != "" { - if err := w.persistOperationalExecutionFailure(executeErr); err != nil { - return err - } - } - return w.reportError("execute prompt", executeErr) -} - -func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error { - startedAt, endedAt := time.Now(), time.Now() - artifact := state.PromptPreparationArtifact{ - SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed, - ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID, - PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, - DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt, - Error: state.NewPromptArtifactError(executeErr), - } - path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact) - if err != nil { - return err - } - w.result.PreparationPath = path - w.metadata.PreparationPath = path - w.result.Metadata = w.metadata - return w.saveMetadata() -} - -func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error { - artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr) - path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact) - if err != nil { - return err - } - w.result.ExecutionPath = path - w.metadata.ExecutionPath = path - w.result.Metadata = w.metadata - return w.saveMetadata() -} - -func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error { - debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution) + path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution) if err != nil { return w.reportError("write prompt debug", promptDebugWriteError(err)) } - if debugPath != "" { - w.result.LLMDebugPath = debugPath + if path != "" { + w.result.LLMDebugPath = path } return nil } -func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error { - rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput) +func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) { + generatedText, _, err := w.handler.Validate(raw) if err != nil { - return err + return w.result, w.reportError("validate generated text", err) } - w.result.GeneratedTextRawPath = rawPath - w.metadata.GeneratedTextRawPath = rawPath - w.result.Metadata = w.metadata - w.executionArtifact = state.PromptExecutionArtifact{ - SchemaVersion: state.PromptExecutionSchemaVersion, - ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID, - PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, - Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation, - Paths: state.PromptExecutionPaths{RawOutputPath: rawPath}, - StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration, - } - if execution.Validation.Status == promptexec.ValidationPassed { - w.executionArtifact.Status = state.PromptExecutionSucceeded - } else { - w.executionArtifact.Status = state.PromptExecutionValidationRejected - } - executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact) - if err != nil { - return err - } - w.result.ExecutionPath = executionPath - w.metadata.ExecutionPath = executionPath - w.result.Metadata = w.metadata - return w.saveMetadata() -} - -func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) { - generatedText, normalized, err := w.handler.Validate(raw) - if err != nil { - return nil, w.reportError("validate generated text", err) - } - generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized) - if err != nil { - return nil, err - } - w.result.GeneratedTextPath = generatedTextPath - w.metadata.GeneratedTextPath = generatedTextPath - w.result.Metadata = w.metadata - if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil { - return nil, err - } - renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText) if err != nil { - return nil, w.reportError("build render context", err) + return w.result, w.reportError("build render context", err) } - renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext) - if err != nil { - return nil, err - } - w.result.RenderContextPath = renderContextPath - w.metadata.RenderContextPath = renderContextPath - w.result.Metadata = w.metadata - if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil { - return nil, err - } - rendered, err := w.handler.Render(renderContext) if err != nil { - return nil, w.reportError("render template", err) + return w.result, w.reportError("render template", err) } - return rendered, nil -} - -func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) { - reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved) + if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil { + return w.result, err + } + w.result.OutputPath = w.req.OutputPath + if w.req.noNotify { + return w.result, nil + } + notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier) + w.result.Notification = notification if err != nil { return w.result, err } - if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil { - return w.result, err - } - w.result.ReportPath = reportPath - w.metadata.RenderedReportPath = reportPath - w.result.Metadata = w.metadata - if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil { - return w.result, err - } - finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{ - Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath, - ExecutionArtifact: &w.executionArtifact, RenderedReportPath: reportPath, OutputPath: w.req.OutputPath, - Notifier: w.req.Notifier, noNotify: w.req.noNotify, - }) - w.result.OutputPath = finalized.OutputPath - w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification - return w.result, err -} - -func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error { - return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update) -} - -func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error { - if err := w.persistReachedPath(update); err != nil { - return err - } - return w.saveMetadata() -} - -func (w *promptReportWorkflow) saveMetadata() error { - path, err := w.store.SaveMetadata(w.ctx, w.metadata) - if err != nil { - return err - } - w.result.Metadata = w.metadata - w.result.MetadataPath = path - return nil + return w.result, nil } func (w *promptReportWorkflow) reportError(operation string, err error) error { - return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err) -} - -func persistReachedPromptPath( - ctx context.Context, - store state.Store, - resolved report.Resolved, - artifact *state.PromptExecutionArtifact, - update func(*state.PromptExecutionPaths), -) error { - update(&artifact.Paths) - _, err := store.SavePromptExecution(ctx, resolved, *artifact) - return err -} - -func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact { - now := time.Now() - return state.PromptExecutionArtifact{ - SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed, - ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID, - PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now, - Error: state.NewPromptArtifactError(err), - } + return generatedReportError(w.req.Resolved, w.result.RunID, operation, err) } func classifiedPromptError(operation string, err error) error { @@ -409,5 +198,3 @@ func classifiedPromptError(operation string, err error) error { func promptDebugWriteError(err error) error { return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err) } - -func ptr[T any](value T) *T { return &value } diff --git a/internal/app/single_report_workflow_test.go b/internal/app/single_report_workflow_test.go deleted file mode 100644 index 147b112..0000000 --- a/internal/app/single_report_workflow_test.go +++ /dev/null @@ -1,671 +0,0 @@ -package app - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" - "gitea.maximumdirect.net/eric/weatherreporter/internal/config" - "gitea.maximumdirect.net/eric/weatherreporter/internal/module" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" - "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" -) - -type workflowCollector struct { - result *collect.Result - err error - calls int -} - -func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) { - c.calls++ - if c.err != nil { - return nil, c.err - } - return c.result, nil -} - -type workflowExecutor struct { - definition report.Definition - raw []byte - prompt promptexec.PromptInspection - inspectionErr error - profile promptexec.ProfileInspection - profileErr error - beforePreparationErr error - afterCallbackErr error - afterPreparationErr error - validation promptexec.ValidationStatus - executeCalls int - providerCalls int - request promptexec.ExecuteRequest - beforeProvider func() - preparationDebug *promptexec.PreparationDebug - executionDebug *promptexec.ExecutionDebug - preparation *promptexec.Preparation - execution *promptexec.Execution -} - -func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { - if e.inspectionErr != nil { - return promptexec.PromptInspection{}, e.inspectionErr - } - if id != e.definition.PromptID || version != e.definition.PromptVersion { - return promptexec.PromptInspection{}, errors.New("unexpected prompt identity") - } - if e.prompt.PromptID != "" { - return e.prompt, nil - } - return validPromptInspection(e.definition), nil -} - -func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { - if e.profileErr != nil { - return promptexec.ProfileInspection{}, e.profileErr - } - profile := e.profile - if profile.ProfileID == "" { - profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"} - } - return profile, nil -} - -func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { - e.executeCalls++ - e.request = req - if e.beforePreparationErr != nil { - return nil, e.beforePreparationErr - } - stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - profile := e.profile - if profile.ProfileID == "" { - profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"} - } - preparation := promptexec.Preparation{ - PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", - RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID, - ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, - } - e.preparation = &preparation - if err := callback(preparation, e.preparationDebug); err != nil { - return nil, err - } - if e.afterCallbackErr != nil { - return nil, e.afterCallbackErr - } - if e.beforeProvider != nil { - e.beforeProvider() - } - e.providerCalls++ - if e.afterPreparationErr != nil { - return nil, e.afterPreparationErr - } - validation := e.validation - if validation == "" { - validation = promptexec.ValidationPassed - } - execution := &promptexec.Execution{ - RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, - PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, - BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash", - StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw, - Debug: e.executionDebug, - Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil), - } - e.execution = execution - return execution, nil -} - -type workflowNotifier struct { - requests []NotificationRequest - err error -} - -func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) { - n.requests = append(n.requests, req) - if n.err != nil { - return nil, n.err - } - return &NotificationResult{ - RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID, - IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted", - }, nil -} - -func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) { - tests := []struct { - name string - kind ReportKind - id report.ID - date time.Time - raw string - wantOutput string - }{ - {name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."}, - {name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."}, - {name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."}, - {name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - definition := report.DefaultRegistry().MustLookup(test.id) - bundle := workflowBundle(t) - collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} - executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)} - notifier := &workflowNotifier{} - outputPath := filepath.Join(t.TempDir(), test.name+".md") - - result, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier, - }) - if err != nil { - t.Fatalf("GenerateDetailed() error = %v", err) - } - if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID { - t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID) - } - if executor.request.PromptVersion != definition.PromptVersion { - t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion) - } - filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace) - if storeErr != nil { - t.Fatalf("NewFilesystemStore() error = %v", storeErr) - } - preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath) - if loadErr != nil || preparation.PromptVersion != definition.PromptVersion { - t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion) - } - if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 { - t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls) - } - persisted, readErr := os.ReadFile(result.DataPackagePath) - if readErr != nil { - t.Fatalf("read data package: %v", readErr) - } - if !bytes.Equal(executor.request.DataPackage, persisted) { - t.Fatal("executor data package differs from exact persisted YAML bytes") - } - renderedReport, readErr := os.ReadFile(result.ReportPath) - if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) { - t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput) - } - copied, readErr := os.ReadFile(outputPath) - if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath { - t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath) - } - if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath { - t.Fatalf("notification requests = %#v, want selected output source", notifier.requests) - } - wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup - if notifier.requests[0].PipelineID != wantPipeline { - t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline) - } - validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02") - wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID) - if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") { - t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths) - } - renderedName := filepath.Base(result.ReportPath) - if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" { - t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath) - } - }) - } -} - -func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) { - tests := []struct { - name string - kind ReportKind - id report.ID - raw string - override string - profile promptexec.ProfileInspection - }{ - { - name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), - profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"}, - }, - { - name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), - profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"}, - }, - { - name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile", - profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - cfg.Promptkit.Profile = test.override - definition := report.DefaultRegistry().MustLookup(test.id) - executor := &workflowExecutor{ - definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw), - } - bundle := workflowBundle(t) - _, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{}, - }) - if err != nil { - t.Fatalf("GenerateDetailed() error = %v", err) - } - if executor.request.ProfileID != test.profile.ProfileID { - t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID) - } - if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName { - t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName) - } - if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName { - t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName) - } - }) - } -} - -type preparationFailingStore struct { - state.Store -} - -type renderContextFailingStore struct { - state.Store -} - -func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) { - path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot) - if err != nil { - return "", err - } - for index := range snapshot.Outputs { - snapshot.Outputs[index].Value = "invalid module value" - } - return path, nil -} - -func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) { - return "", errors.New("injected preparation persistence failure") -} - -func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) { - tests := []struct { - name string - configure func(*workflowExecutor) - wantCategory promptexec.ErrorCategory - wantPreparation bool - wantExecution bool - wantRaw bool - wantProviderCall int - }{ - {name: "preparation", configure: func(e *workflowExecutor) { - e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil) - }, wantCategory: promptexec.Generation, wantPreparation: true}, - {name: "credential disappears", configure: func(e *workflowExecutor) { - e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil) - }, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true}, - {name: "capacity is not retried", configure: func(e *workflowExecutor) { - e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil) - }, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "canceled", configure: func(e *workflowExecutor) { - e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled) - }, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "deadline", configure: func(e *workflowExecutor) { - e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded) - }, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "generation", configure: func(e *workflowExecutor) { - e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil) - }, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "operational validation error", configure: func(e *workflowExecutor) { - e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil) - }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1}, - {name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = false - definition := report.DefaultRegistry().MustLookup(report.Daily) - executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())} - test.configure(executor) - bundle := workflowBundle(t) - collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}} - result, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - Collector: collector, Executor: executor, - }) - if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory { - t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory) - } - if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw { - t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath) - } - if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall { - t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall) - } - }) - } -} - -func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) { - tests := []struct { - name string - configure func(*workflowExecutor) - wantCategory promptexec.ErrorCategory - }{ - {name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration}, - {name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration}, - {name: "malformed profile", configure: func(e *workflowExecutor) { - e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret") - }, wantCategory: promptexec.InvalidConfiguration}, - {name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration}, - {name: "credential", configure: func(e *workflowExecutor) { - e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true} - }, wantCategory: promptexec.MissingCredential}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - definition := report.DefaultRegistry().MustLookup(report.Daily) - executor := &workflowExecutor{definition: definition} - test.configure(executor) - collector := &workflowCollector{err: errors.New("collector must not run")} - result, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - Collector: collector, Executor: executor, - }) - if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 { - t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls) - } - if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") { - t.Fatalf("error leaks profile details: %v", err) - } - entries, readErr := os.ReadDir(cfg.Workspace.Root) - if readErr != nil || len(entries) != 0 { - t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr) - } - }) - } -} - -func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) { - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = false - filesystem, err := state.NewFilesystemStore(cfg.Workspace) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - definition := report.DefaultRegistry().MustLookup(report.Daily) - executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())} - bundle := workflowBundle(t) - result, err := GenerateDetailed(context.Background(), GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem}, - }) - if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 { - t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls) - } -} - -func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) { - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = false - now := workflowTime("2026-05-29T08:30:00-05:00") - request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()} - resolved, err := ResolveGenerate(request, now) - if err != nil { - t.Fatalf("ResolveGenerate() error = %v", err) - } - filesystem, err := state.NewFilesystemStore(cfg.Workspace) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - paths, err := filesystem.Paths(resolved) - if err != nil { - t.Fatalf("Paths() error = %v", err) - } - checked := false - executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())} - executor.beforeProvider = func() { - checked = true - if _, statErr := os.Stat(paths.Preparation); statErr != nil { - t.Fatalf("preparation was not durable before provider execution: %v", statErr) - } - } - bundle := workflowBundle(t) - request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}} - request.Executor = executor - request.Store = filesystem - result, err := GenerateDetailed(context.Background(), request) - if err != nil || result == nil || !checked || result.OutputPath == "" { - t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath) - } -} - -func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) { - tests := []struct { - name string - raw string - configure func(*GenerateRequest, *workflowNotifier) - wantRaw bool - wantNormalized bool - wantContext bool - wantReport bool - wantOutput bool - }{ - {name: "generated text decode", raw: `{`, wantRaw: true}, - {name: "generated text domain", raw: `{}`, wantRaw: true}, - {name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { - req.Store = renderContextFailingStore{Store: req.Store} - }, wantRaw: true, wantNormalized: true}, - {name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { - req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext} - }, wantRaw: true, wantNormalized: true}, - {name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) { - blocker := filepath.Join(t.TempDir(), "report-blocker") - if err := os.Mkdir(blocker, 0o700); err != nil { - t.Fatalf("create report blocker: %v", err) - } - req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker} - }, wantRaw: true, wantNormalized: true, wantContext: true}, - {name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) { - notifier.err = errors.New("notification rejected") - }, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - definition := report.DefaultRegistry().MustLookup(report.Daily) - executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)} - notifier := &workflowNotifier{} - bundle := workflowBundle(t) - filesystem, err := state.NewFilesystemStore(cfg.Workspace) - if err != nil { - t.Fatalf("NewFilesystemStore() error = %v", err) - } - req := GenerateRequest{ - Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), - OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, - Executor: executor, Notifier: notifier, Store: filesystem, - } - if test.configure != nil { - test.configure(&req, notifier) - } - result, err := GenerateDetailed(context.Background(), req) - if err == nil || result == nil { - t.Fatalf("result/error = %#v/%v, want partial result and error", result, err) - } - if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized || - (result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport || - (result.OutputPath != "") != test.wantOutput { - t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath) - } - if test.wantRaw { - persisted, readErr := os.ReadFile(result.GeneratedTextRawPath) - if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) { - t.Fatalf("retained raw output = %q, error %v", persisted, readErr) - } - } - if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) { - t.Fatalf("notification requests = %#v", notifier.requests) - } - if test.name == "notification" && result.Metadata.NotificationPath != "" { - t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata) - } - }) - } -} - -func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) { - tests := []struct { - name string - createCollision func(string, report.Resolved) error - wantProviderCalls int - wantPreparationFile bool - }{ - { - name: "preparation debug", - createCollision: func(root string, resolved report.Resolved) error { - path := workflowDebugRunPath(root, resolved) - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - return os.WriteFile(path, []byte("not a directory"), 0o600) - }, - }, - { - name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true, - createCollision: func(root string, resolved report.Resolved) error { - return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700) - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := workflowConfig(t) - cfg.Notify.Distributor.Enabled = false - now := workflowTime("2026-05-29T08:30:00-05:00") - request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()} - resolved, err := ResolveGenerate(request, now) - if err != nil { - t.Fatalf("ResolveGenerate() error = %v", err) - } - debugRoot := filepath.Join(t.TempDir(), "prompt-debug") - if err := test.createCollision(debugRoot, resolved); err != nil { - t.Fatalf("create debug collision: %v", err) - } - bundle := workflowBundle(t) - executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())} - request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}} - request.Executor = executor - request.LLMDebugDir = debugRoot - result, err := GenerateDetailed(context.Background(), request) - if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { - t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err)) - } - if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" { - t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath) - } - if test.wantPreparationFile && result.MetadataPath == "" { - t.Fatal("execution debug failure lost previously persisted metadata") - } - preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json") - _, statErr := os.Stat(preparationDebug) - if (statErr == nil) != test.wantPreparationFile { - t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile) - } - }) - } -} - -func workflowDebugRunPath(root string, resolved report.Resolved) string { - return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID) -} - -func workflowBundlePaths(id report.ID, validDate, runID string) []string { - switch id { - case report.Daily: - return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"} - case report.Today: - return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"} - case report.Tomorrow: - return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"} - case report.Hourly: - return []string{"hourly/index.md"} - default: - return nil - } -} - -func workflowConfig(t *testing.T) config.Config { - t.Helper() - cfg := config.Defaults() - cfg.Workspace.Root = t.TempDir() - cfg.WeatherAPI.Timezone = "America/Chicago" - cfg.Location.ID = "home" - cfg.Location.Name = "Testville" - cfg.Location.Region = "MO" - cfg.Notify.Distributor.Enabled = true - cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}" - return cfg -} - -func workflowBundle(t *testing.T) weatherdata.Bundle { - t.Helper() - data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json")) - if err != nil { - t.Fatalf("read bundle fixture: %v", err) - } - var bundle weatherdata.Bundle - if err := json.Unmarshal(data, &bundle); err != nil { - t.Fatalf("decode bundle fixture: %v", err) - } - future := bundle.Hourly.Periods[0] - future.StartTime = workflowTime("2026-05-30T06:00:00-05:00") - future.EndTime = workflowTime("2026-05-30T07:00:00-05:00") - bundle.Hourly.Periods = append(bundle.Hourly.Periods, future) - futureNarrative := bundle.Narrative.Periods[0] - futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00") - futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00") - futureNarrative.Name = "Tomorrow" - bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative) - return bundle -} - -func workflowTime(value string) time.Time { - parsed, err := time.Parse(time.RFC3339, value) - if err != nil { - panic(err) - } - return parsed -} - -func validHourlyWorkflowJSON() string { - return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}` -} - -func validTomorrowWorkflowJSON() string { - return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}` -} - -func validTodayWorkflowJSON() string { - return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}` -} - -func validDailyWorkflowJSON() string { - return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}` -} diff --git a/internal/cli/result.go b/internal/cli/result.go index f7a9904..6fad627 100644 --- a/internal/cli/result.go +++ b/internal/cli/result.go @@ -6,6 +6,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/app" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) const ( @@ -17,26 +18,25 @@ const ( ) type generateSummary struct { - Command string `json:"command"` - ReportID report.ID `json:"reportId"` - ReportName string `json:"reportName"` - PromptID string `json:"promptId"` - RunID string `json:"runId"` - Status string `json:"status"` - GeneratedAt time.Time `json:"generatedAt"` - ValidPeriod timeutil.Period `json:"validPeriod"` - ReportPath string `json:"reportPath,omitempty"` - OutputPath string `json:"outputPath,omitempty"` - MetadataPath string `json:"metadataPath,omitempty"` - DataPackagePath string `json:"dataPackagePath,omitempty"` - PreparationPath string `json:"preparationPath,omitempty"` - ExecutionPath string `json:"executionPath,omitempty"` - LLMDebugPath string `json:"llmDebugPath,omitempty"` - GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"` - GeneratedTextPath string `json:"generatedTextPath,omitempty"` - RenderContextPath string `json:"renderContextPath,omitempty"` - Notification *generateNotificationSummary `json:"notification,omitempty"` - Error string `json:"error,omitempty"` + Command string `json:"command"` + ReportID report.ID `json:"reportId"` + ReportName string `json:"reportName"` + PromptID string `json:"promptId"` + RunID string `json:"runId"` + Status string `json:"status"` + GeneratedAt time.Time `json:"generatedAt"` + ValidPeriod timeutil.Period `json:"validPeriod"` + OutputPath string `json:"outputPath,omitempty"` + LLMDebugPath string `json:"llmDebugPath,omitempty"` + PromptVersion string `json:"promptVersion"` + Timezone string `json:"timezone"` + ProfileID string `json:"profileId,omitempty"` + BackendID string `json:"backendId,omitempty"` + ModelName string `json:"modelName,omitempty"` + SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"` + ValidationStatus string `json:"validationStatus,omitempty"` + Notification *generateNotificationSummary `json:"notification,omitempty"` + Error string `json:"error,omitempty"` } type generateNotificationSummary struct { @@ -73,24 +73,20 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary { return summary } - metadata := result.Metadata - summary.ReportID = metadata.ReportID - summary.ReportName = reportName(metadata.ReportID) - summary.PromptID = metadata.PromptID - summary.RunID = metadata.RunID + summary.ReportID = result.ReportID + summary.ReportName = result.ReportName + summary.PromptID = result.PromptID + summary.PromptVersion = result.PromptVersion + summary.RunID = result.RunID summary.Status = summaryStatusSucceeded - summary.GeneratedAt = metadata.GeneratedAt - summary.ValidPeriod = metadata.ValidPeriod - summary.ReportPath = result.ReportPath + summary.GeneratedAt = result.GeneratedAt + summary.ValidPeriod = result.ValidPeriod + summary.Timezone = result.Timezone + summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName + summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...) + summary.ValidationStatus = string(result.ValidationStatus) summary.OutputPath = result.OutputPath - summary.MetadataPath = result.MetadataPath - summary.DataPackagePath = result.DataPackagePath - summary.PreparationPath = result.PreparationPath - summary.ExecutionPath = result.ExecutionPath summary.LLMDebugPath = result.LLMDebugPath - summary.GeneratedTextRawPath = result.GeneratedTextRawPath - summary.GeneratedTextPath = result.GeneratedTextPath - summary.RenderContextPath = result.RenderContextPath summary.Notification = newGenerateNotificationSummary(result.Notification) if err != nil { summary.Status = summaryStatusFailed diff --git a/internal/cli/result_test.go b/internal/cli/result_test.go index d64b4d6..b1c695f 100644 --- a/internal/cli/result_test.go +++ b/internal/cli/result_test.go @@ -2,264 +2,29 @@ package cli import ( "encoding/json" - "errors" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/app" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" ) -func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) { +func TestGenerateSummaryUsesActiveResultFields(t *testing.T) { generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) - acceptedAt := generatedAt.Add(time.Minute) - startedAt := acceptedAt.Add(time.Minute) - finishedAt := startedAt.Add(time.Minute) - result := &app.ReportResult{ - DataPackagePath: "/runs/hourly/data_package.yaml", - PreparationPath: "/runs/hourly/preparation.json", - ExecutionPath: "/runs/hourly/execution.json", - LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123", - ReportPath: "/runs/hourly/report.md", - OutputPath: "/copies/hourly.md", - MetadataPath: "/runs/hourly/metadata.json", - GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json", - GeneratedTextPath: "/runs/hourly/generated_text.json", - RenderContextPath: "/runs/hourly/render_context.json", - Metadata: state.Metadata{ - ReportID: report.Hourly, - PromptID: "weather.hourly_generated_text", - RunID: "20260529T133000Z_hourly", - GeneratedAt: generatedAt, - ValidPeriod: testSummaryPeriod(generatedAt), - }, - Notification: &app.NotificationResult{ - Status: "succeeded", - UploadStatus: "accepted", - RunID: "distributor-run", - PipelineID: "weatherreporter.hourly", - BundleID: "weatherreporter.home.hourly", - IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly", - AcceptedAt: acceptedAt, - StartedAt: &startedAt, - FinishedAt: &finishedAt, - Report: []byte(`{"actions":[{"action":"replace_older"}]}`), - }, - } - - summary := newGenerateSummary(result, nil) - - if summary.Command != "generate" || summary.Status != "succeeded" { - t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status) - } - if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" { - t.Fatalf("summary identity = %#v, want hourly report identity", summary) - } - if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" { - t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary) - } - if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) { - t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification) + summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil) + if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) { + t.Fatalf("summary = %#v", summary) } data, err := json.Marshal(summary) if err != nil { t.Fatalf("Marshal() error = %v", err) } - if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") { - t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data)) - } - if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") { - t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data)) - } -} - -func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) { - generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) - result := &app.ReportResult{ - DataPackagePath: "/runs/daily/data_package.yaml", - PreparationPath: "/runs/daily/preparation.json", - ReportPath: "/runs/daily/report.md", - OutputPath: "/copies/daily.md", - MetadataPath: "/runs/daily/metadata.json", - Metadata: state.Metadata{ - ReportID: report.Daily, - PromptID: "weather.daily_generated_text", - RunID: "20260529T133000Z_daily", - GeneratedAt: generatedAt, - ValidPeriod: testSummaryPeriod(generatedAt), - }, - } - - summary := newGenerateSummary(result, nil) - - if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" { - t.Fatalf("summary = %#v, want successful daily summary", summary) - } - if summary.Notification != nil { - t.Fatalf("notification summary = %#v, want omitted", summary.Notification) - } - data, err := json.Marshal(summary) - if err != nil { - t.Fatalf("Marshal() error = %v", err) - } - for _, omitted := range []string{"notification"} { - if strings.Contains(string(data), omitted) { - t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data)) + for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} { + if strings.Contains(string(data), forbidden) { + t.Fatalf("summary includes %q: %s", forbidden, data) } } } - -func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) { - result := &app.ReportResult{ - DataPackagePath: "/runs/daily/data_package.yaml", - PreparationPath: "/runs/daily/preparation.json", - Metadata: state.Metadata{ - ReportID: report.Daily, - RunID: "20260529T133000Z_daily", - }, - } - - data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed"))) - if err != nil { - t.Fatalf("Marshal() error = %v", err) - } - text := string(data) - for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} { - if strings.Contains(text, omitted) { - t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text) - } - } - if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") { - t.Fatalf("partial summary omits reached paths:\n%s", text) - } -} - -func TestNewGenerateSummaryForNotificationFailure(t *testing.T) { - generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) - result := &app.ReportResult{ - DataPackagePath: "/runs/hourly/data_package.yaml", - PreparationPath: "/runs/hourly/preparation.json", - ReportPath: "/runs/hourly/report.md", - OutputPath: "/copies/hourly.md", - MetadataPath: "/runs/hourly/metadata.json", - Metadata: state.Metadata{ - ReportID: report.Hourly, - PromptID: "weather.hourly_generated_text", - RunID: "20260529T133000Z_hourly", - GeneratedAt: generatedAt, - ValidPeriod: testSummaryPeriod(generatedAt), - }, - } - err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`) - - summary := newGenerateSummary(result, err) - - if summary.Status != "failed" || summary.Error != err.Error() { - t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error) - } - if summary.ReportPath == "" || summary.MetadataPath == "" { - t.Fatalf("artifact paths = report %q metadata %q, want retained output provenance", summary.ReportPath, summary.MetadataPath) - } - data, marshalErr := json.Marshal(summary) - if marshalErr != nil { - t.Fatalf("Marshal() error = %v", marshalErr) - } - if strings.Contains(string(data), "notificationPath") { - t.Fatalf("summary JSON includes a notification receipt path:\n%s", data) - } -} - -func TestNewBatchSummaryStatusDerivation(t *testing.T) { - startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC) - finishedAt := startedAt.Add(2 * time.Minute) - tests := []struct { - name string - result *app.BatchResult - wantStatus string - wantError string - }{ - { - name: "success", - result: &app.BatchResult{ - Batch: app.BatchMorning, - StartedAt: startedAt, - FinishedAt: finishedAt, - Total: 1, - Succeeded: 1, - Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}}, - }, - wantStatus: "succeeded", - }, - { - name: "report failure", - result: &app.BatchResult{ - Batch: app.BatchMorning, - Total: 2, - Succeeded: 1, - Failed: 1, - Reports: []app.BatchReportResult{ - {ReportID: report.Today, Status: "succeeded"}, - {ReportID: report.Tomorrow, Status: "failed", Error: "render failed"}, - }, - }, - wantStatus: "failed", - wantError: "batch morning failed: 1 of 2 reports failed", - }, - { - name: "skipped notification", - result: &app.BatchResult{ - Batch: app.BatchEvening, - Total: 2, - Succeeded: 1, - Failed: 1, - Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}}, - Notification: &app.BatchNotificationResult{ - Status: "skipped", - Reason: "one or more reports failed", - }, - }, - wantStatus: "failed", - wantError: "batch evening failed: 1 of 2 reports failed", - }, - { - name: "failed notification", - result: &app.BatchResult{ - Batch: app.BatchEvening, - Total: 1, - Succeeded: 1, - Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}}, - Notification: &app.BatchNotificationResult{ - Status: "failed", - Error: "notify batch evening: upload rejected", - }, - }, - wantStatus: "failed", - wantError: "batch evening notification failed: notify batch evening: upload rejected", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - summary := newBatchSummary(tt.result) - if summary.Command != "run" || summary.Status != tt.wantStatus { - t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus) - } - if summary.Error != tt.wantError { - t.Fatalf("error = %q, want %q", summary.Error, tt.wantError) - } - if len(summary.Reports) != len(tt.result.Reports) { - t.Fatalf("reports = %#v, want copied report list", summary.Reports) - } - }) - } -} - -func testSummaryPeriod(start time.Time) timeutil.Period { - return timeutil.Period{ - Start: start, - End: start.Add(6 * time.Hour), - } -} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go deleted file mode 100644 index 538ccbc..0000000 --- a/internal/cli/root_test.go +++ /dev/null @@ -1,631 +0,0 @@ -package cli - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/app" - "gitea.maximumdirect.net/eric/weatherreporter/internal/module" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" - "gitea.maximumdirect.net/eric/weatherreporter/internal/state" - "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" -) - -const ( - testRenderedPrompt = "PRIVATE RENDERED PROMPT" - testSchemaBody = `{"private":"schema"}` - testDataBody = "PRIVATE DATA PACKAGE" - testGeneratedBody = "PRIVATE GENERATED BODY" - testEndpoint = "https://user:credential@example.invalid/v1?token=credential" - testParameters = `{"temperature":0.2,"private":"parameter"}` - testCredential = "cli-secret-credential" -) - -type commandOutput struct { - stdout string - stderr string -} - -type cliExecutor struct { - fail bool - failPrompt string -} - -func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { - name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text") - return promptexec.PromptInspection{ - PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile", - Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, - Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"}, - }, nil -} - -func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { - return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil -} - -func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { - now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC) - preparation := promptexec.Preparation{ - PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", - ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath, - StartedAt: now, EndedAt: now, - } - if err := callback(preparation, nil); err != nil { - return nil, err - } - if e.fail || req.PromptID == e.failPrompt { - return nil, errors.New(strings.Join([]string{ - "provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters, - }, " ")) - } - raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`) - if req.PromptID == "weather.hourly_generated_text" { - raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`) - } - return &promptexec.Execution{ - RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", - ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash", - StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw, - Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil), - }, nil -} - -func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) { - output, err := runCLICommand(Runner{}, "--help") - if err != nil { - t.Fatalf("Run(--help) error = %v", err) - } - for _, command := range []string{ - "--version", - "generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening", - "inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources", - } { - if !strings.Contains(output.stdout, command) { - t.Fatalf("help missing %q:\n%s", command, output.stdout) - } - } - for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} { - if strings.Contains(output.stdout, retired) { - t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout) - } - } - for _, description := range []string{"Write the generated Markdown report to PATH.", "Write generated Markdown reports beneath PATH"} { - if !strings.Contains(output.stdout, description) { - t.Fatalf("help missing output description %q:\n%s", description, output.stdout) - } - } -} - -func TestRunnerVersion(t *testing.T) { - for _, test := range []struct { - name string - runner Runner - version string - }{ - {name: "development default", runner: Runner{}, version: "development"}, - {name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"}, - } { - t.Run(test.name, func(t *testing.T) { - output, err := runCLICommand(test.runner, "--version") - if err != nil { - t.Fatalf("Run(--version) error = %v", err) - } - if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" { - t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr) - } - }) - } - if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil { - t.Fatal("Run(--version extra) error = nil") - } -} - -func TestResolveSupportedCommandsAndFlags(t *testing.T) { - configPath := writeCLIConfig(t, t.TempDir(), "") - runner, constructions := countingRunner(cliExecutor{}) - - for _, tt := range []struct { - name string - args []string - want app.ReportKind - }{ - {name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily}, - {name: "today", args: []string{"today"}, want: app.ReportToday}, - {name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow}, - {name: "hourly", args: []string{"hourly"}, want: app.ReportHourly}, - } { - t.Run(tt.name, func(t *testing.T) { - args := append(tt.args, "--config", configPath) - req, err := runner.resolveGenerate(args) - if err != nil { - t.Fatalf("resolveGenerate() error = %v", err) - } - if req.Report != tt.want || req.Executor == nil { - t.Fatalf("request = %#v, want report %q with executor", req, tt.want) - } - if tt.want == app.ReportDaily || tt.want == app.ReportToday { - if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" { - t.Fatalf("resolved date = %q, want 2026-05-29", got) - } - } else if !req.Date.IsZero() { - t.Fatalf("resolved date = %s, want unset", req.Date) - } - }) - } - - for _, tt := range []struct { - name string - want app.BatchKind - }{ - {name: "morning", want: app.BatchMorning}, - {name: "evening", want: app.BatchEvening}, - } { - t.Run(tt.name, func(t *testing.T) { - req, err := runner.resolveRun([]string{tt.name, "--config", configPath}) - if err != nil { - t.Fatalf("resolveRun() error = %v", err) - } - if req.Batch != tt.want || req.Executor == nil { - t.Fatalf("request = %#v, want batch %q with executor", req, tt.want) - } - }) - } - if *constructions != 6 { - t.Fatalf("executor constructions = %d, want one per resolved action", *constructions) - } -} - -func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) { - configPath := writeCLIConfig(t, t.TempDir(), "") - runner, _ := countingRunner(cliExecutor{}) - runner.WorkingDir = t.TempDir() - - generate, generateOpts, err := runner.resolveGenerateAction([]string{ - "daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC", - "--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet", - }) - if err != nil { - t.Fatalf("resolveGenerateAction() error = %v", err) - } - if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != filepath.Join(runner.WorkingDir, "daily.md") || generate.WorkingDir != runner.WorkingDir || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet { - t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts) - } - if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" { - t.Fatalf("generate date = %q, want 2026-05-30", got) - } - - batch, batchOpts, err := runner.resolveRunAction([]string{ - "evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports", - "--llm-debug-dir", "/safe/debug", "--quiet", - }) - if err != nil { - t.Fatalf("resolveRunAction() error = %v", err) - } - if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != filepath.Join(runner.WorkingDir, "reports") || batch.WorkingDir != runner.WorkingDir || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet { - t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts) - } -} - -func TestGenerateFlagContracts(t *testing.T) { - for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} { - t.Run(string(kind), func(t *testing.T) { - opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"}) - if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" { - t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err) - } - }) - } - - runner, _ := countingRunner(cliExecutor{}) - for _, tt := range []struct { - name string - args []string - want string - }{ - {name: "daily requires date", args: []string{"daily"}, want: "requires --date"}, - {name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"}, - {name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"}, - {name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}}, - {name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}}, - {name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}}, - {name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}}, - {name: "batch rejects output", args: []string{"run", "--out", "report.md"}}, - } { - t.Run(tt.name, func(t *testing.T) { - var err error - if tt.args[0] == "run" { - _, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...)) - } else { - _, err = runner.resolveGenerate(tt.args) - } - if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) { - t.Fatalf("error = %v, want rejection containing %q", err, tt.want) - } - }) - } -} - -func TestResolversRejectRetiredAndUnknownNames(t *testing.T) { - runner, _ := countingRunner(cliExecutor{}) - for _, name := range []string{"near-term", "three-day", "weekend", "storm"} { - if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") { - t.Fatalf("resolveGenerate(%q) error = %v", name, err) - } - } - for _, name := range []string{"daily", "weekend", "storm"} { - if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") { - t.Fatalf("resolveRun(%q) error = %v", name, err) - } - } -} - -func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) { - for _, tt := range []struct { - name string - args func(string, string) []string - }{ - {name: "single", args: func(configPath, outputPath string) []string { - return []string{"generate", "today", "--config", configPath, "--out", outputPath} - }}, - {name: "batch", args: func(configPath, outputPath string) []string { - return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath} - }}, - } { - t.Run(tt.name, func(t *testing.T) { - fixture := newCLIFixture(t) - runner, constructions := countingRunner(cliExecutor{}) - runner.WorkingDir = t.TempDir() - output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if *constructions != 1 { - t.Fatalf("executor constructions = %d, want 1", *constructions) - } - assertRoutineOutputSafe(t, output) - if tt.name == "single" { - summary := decodeGenerateSummary(t, output.stdout) - if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" { - t.Fatalf("single summary = %#v", summary) - } - if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) { - t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout) - } - } else { - summary := decodeBatchSummary(t, output.stdout) - if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" { - t.Fatalf("batch summary = %#v", summary) - } - if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") { - t.Fatalf("batch status = %q", output.stderr) - } - if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) { - t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout) - } - } - }) - } -} - -func TestRunnerPreRunFailureAndQuietMode(t *testing.T) { - runner, constructions := countingRunner(cliExecutor{}) - runner.WorkingDir = t.TempDir() - output, err := runCLICommand(runner, "generate", "daily") - if err == nil || output.stdout != "" || output.stderr != "" { - t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err) - } - if *constructions != 1 { - t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions) - } - - fixture := newCLIFixture(t) - runner, _ = countingRunner(cliExecutor{}) - runner.WorkingDir = t.TempDir() - output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet") - if err != nil || output.stdout != "" || output.stderr != "" { - t.Fatalf("quiet output/error = %#v/%v", output, err) - } -} - -func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) { - fixture := newCLIFixture(t) - runner, constructions := countingRunner(cliExecutor{fail: true}) - runner.WorkingDir = t.TempDir() - output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath) - if err == nil { - t.Fatal("Run() error = nil, want execution failure") - } - if *constructions != 1 { - t.Fatalf("executor constructions = %d, want 1", *constructions) - } - summary := decodeGenerateSummary(t, output.stdout) - if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" { - t.Fatalf("failed summary paths = %#v", summary) - } - if !strings.Contains(summary.Error, "prompt execution failed") { - t.Fatalf("failed summary error = %q", summary.Error) - } - assertRoutineOutputSafe(t, output) - - for _, command := range []string{"metadata", "modules", "data-package", "sources"} { - inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID) - if inspectErr != nil { - t.Fatalf("inspect %s error = %v", command, inspectErr) - } - if !strings.Contains(inspected.stdout, summary.RunID) { - t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout) - } - assertRoutineOutputSafe(t, inspected) - } -} - -func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) { - fixture := newCLIFixture(t) - runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"}) - runner.WorkingDir = t.TempDir() - output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath) - if err == nil { - t.Fatal("Run() error = nil, want aggregate batch failure") - } - if *constructions != 1 { - t.Fatalf("executor constructions = %d, want 1", *constructions) - } - summary := decodeBatchSummary(t, output.stdout) - if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 { - t.Fatalf("failed batch summary = %#v", summary) - } - var succeeded, failed *app.BatchReportResult - for index := range summary.Reports { - item := &summary.Reports[index] - if item.Status == summaryStatusSucceeded { - succeeded = item - } else if item.Status == summaryStatusFailed { - failed = item - } - } - if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" { - t.Fatalf("successful batch item paths = %#v", succeeded) - } - if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" { - t.Fatalf("failed batch item paths = %#v", failed) - } - if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") { - t.Fatalf("partial batch status = %q", output.stderr) - } - assertRoutineOutputSafe(t, output) -} - -func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) { - fixture := newCLIFixture(t) - runner, _ := countingRunner(cliExecutor{}) - runner.WorkingDir = t.TempDir() - first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC)) - second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)) - - listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2") - if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) { - t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err) - } - for _, command := range []string{"metadata", "modules", "data-package", "sources"} { - output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID) - if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) { - t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr) - } - assertRoutineOutputSafe(t, output) - } - prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID) - if err != nil || !strings.Contains(prior.stdout, first.RunID) { - t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err) - } -} - -func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) { - fixture := newCLIFixture(t) - runIDs := []string{ - writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)), - writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)), - writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)), - } - runID := runIDs[0] - runner, _ := countingRunner(cliExecutor{}) - - listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath) - if err != nil { - t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err) - } - for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} { - if !strings.Contains(listed.stdout, want) { - t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout) - } - } - for _, command := range []string{"metadata", "modules", "data-package", "sources"} { - output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID) - if inspectErr != nil || !strings.Contains(output.stdout, runID) { - t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr) - } - } - metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID) - if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) { - t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err) - } -} - -func countingRunner(executor promptexec.Executor) (Runner, *int) { - count := new(int) - return Runner{ - Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)}, - ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { - *count++ - return executor, nil - }, - }, count -} - -func runCLICommand(runner Runner, args ...string) (commandOutput, error) { - var stdout bytes.Buffer - var stderr bytes.Buffer - err := runner.Run(context.Background(), args, &stdout, &stderr) - return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err -} - -func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary { - t.Helper() - base.Clock = timeutil.FixedClock{Time: now} - output, err := runCLICommand(base, "generate", "today", "--config", configPath) - if err != nil { - t.Fatalf("generate current report: %v", err) - } - return decodeGenerateSummary(t, output.stdout) -} - -func decodeGenerateSummary(t *testing.T, text string) generateSummary { - t.Helper() - var summary generateSummary - if err := json.Unmarshal([]byte(text), &summary); err != nil { - t.Fatalf("decode generate summary: %v\n%s", err, text) - } - return summary -} - -func decodeBatchSummary(t *testing.T, text string) batchSummary { - t.Helper() - var summary batchSummary - if err := json.Unmarshal([]byte(text), &summary); err != nil { - t.Fatalf("decode batch summary: %v\n%s", err, text) - } - return summary -} - -func assertRoutineOutputSafe(t *testing.T, output commandOutput) { - t.Helper() - combined := output.stdout + output.stderr - for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} { - if strings.Contains(combined, forbidden) { - t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined) - } - } -} - -type cliFixture struct { - tempDir string - workspaceRoot string - configPath string -} - -func newCLIFixture(t *testing.T) cliFixture { - t.Helper() - tempDir := t.TempDir() - workspaceRoot := filepath.Join(tempDir, "workspace") - server := weatherServer(t) - return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")} -} - -func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) } - -func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string { - t.Helper() - configPath := filepath.Join(t.TempDir(), "config.yml") - body := "weather_api:\n timezone: America/Chicago\n" - if baseURL != "" { - body += " base_url: " + baseURL + "\n" - } - body += "workspace:\n root: " + workspaceRoot + "\n" - if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - return configPath -} - -func weatherServer(t *testing.T) *httptest.Server { - t.Helper() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/observations": - _, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`)) - case "/conditions/current": - _, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`)) - case "/forecast/hourly": - _, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`)) - case "/forecast/narrative": - _, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`)) - case "/alerts/active": - _, _ = w.Write([]byte(`{"data":{"alerts":[]}}`)) - case "/discussion": - _, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`)) - case "/weatherstories/latest": - _, _ = w.Write([]byte(`{"data":null}`)) - case "/outlooks/convective": - _, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`)) - default: - http.NotFound(w, r) - } - })) - t.Cleanup(server.Close) - return server -} - -func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string { - t.Helper() - runID := "historical-" + string(reportID) - date := generatedAt.Format(timeutil.DateLayout) - dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date) - modulePath := filepath.Join(dir, "modules."+runID+".json") - dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml") - metadataPath := filepath.Join(dir, "metadata."+runID+".json") - if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil { - t.Fatalf("create historical fixture directory: %v", err) - } - if err := os.MkdirAll(dir, 0o755); err != nil { - t.Fatalf("create historical metadata directory: %v", err) - } - snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}}) - if err != nil { - t.Fatalf("build historical module snapshot: %v", err) - } - moduleData, err := json.Marshal(snapshot) - if err != nil { - t.Fatalf("marshal historical module snapshot: %v", err) - } - if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil { - t.Fatalf("write historical module snapshot: %v", err) - } - period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)} - pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{ - RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, - }, Modules: snapshot}) - if err != nil { - t.Fatalf("build historical data package: %v", err) - } - data, err := promptinput.MarshalYAML(pkg) - if err != nil { - t.Fatalf("marshal historical data package: %v", err) - } - if err := os.WriteFile(dataPath, data, 0o600); err != nil { - t.Fatalf("write historical data package: %v", err) - } - metadata := state.Metadata{ - SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), - GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive", - ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json", - } - metadataData, err := json.Marshal(metadata) - if err != nil { - t.Fatalf("marshal historical metadata: %v", err) - } - if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil { - t.Fatalf("write historical metadata: %v", err) - } - return runID -} diff --git a/internal/promptdebug/debug_writer.go b/internal/promptdebug/debug_writer.go index a25166f..5b37c19 100644 --- a/internal/promptdebug/debug_writer.go +++ b/internal/promptdebug/debug_writer.go @@ -16,8 +16,8 @@ import ( ) const ( - promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v1" - promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v1" + promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2" + promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2" debugDirectoryMode = 0o700 debugFileMode = 0o600 ) @@ -64,7 +64,6 @@ type PromptDebugPreparation struct { StartedAt time.Time `json:"startedAt"` EndedAt time.Time `json:"endedAt"` Duration time.Duration `json:"duration"` - DataPackagePath string `json:"dataPackagePath"` } // PromptPreparationDebugArtifact is the on-disk preparation debug record. @@ -114,7 +113,6 @@ type PromptDebugExecution struct { StartedAt time.Time `json:"startedAt"` EndedAt time.Time `json:"endedAt"` Duration time.Duration `json:"duration"` - DataPackagePath string `json:"dataPackagePath"` } // PromptExecutionDebugArtifact is the on-disk execution debug record. @@ -340,7 +338,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName, Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath}, - StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath, + StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, } } @@ -351,7 +349,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution { InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens}, - StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath, + StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, } } diff --git a/internal/promptdebug/debug_writer_test.go b/internal/promptdebug/debug_writer_test.go index 3e694df..e0e09f6 100644 --- a/internal/promptdebug/debug_writer_test.go +++ b/internal/promptdebug/debug_writer_test.go @@ -40,19 +40,19 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) { t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) } preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) - for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} { + for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} { if !strings.Contains(string(preparationData), want) { t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) } } executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json")) - for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} { + for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} { if !strings.Contains(string(executionData), want) { t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData) } } for _, data := range [][]byte{preparationData, executionData} { - if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") { + if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") { t.Fatalf("debug artifact contains credentials:\n%s", data) } } @@ -169,7 +169,7 @@ func promptDebugPreparationFixture() promptexec.Preparation { PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"}, - StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml", + StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, } } @@ -180,9 +180,9 @@ func promptDebugExecutionFixture() promptexec.Execution { InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, - Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}), - DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."), - Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}}, + Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}), + RawOutput: []byte("Generated forecast prose."), + Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}}, } } diff --git a/internal/promptexec/promptexec.go b/internal/promptexec/promptexec.go index ee6791a..30a691d 100644 --- a/internal/promptexec/promptexec.go +++ b/internal/promptexec/promptexec.go @@ -61,15 +61,13 @@ type ProfileInspection struct { } // ExecuteRequest selects one exact prompt execution. DataPackage is the exact -// YAML input; implementations must copy it before retaining it. DataPackagePath -// is provenance for the inline input, not a provider-readable file reference. +// YAML input; implementations must copy it before retaining it. type ExecuteRequest struct { - PromptID string - PromptVersion string - ProfileID string - DataPackage []byte - DataPackagePath string - CaptureDebug bool + PromptID string + PromptVersion string + ProfileID string + DataPackage []byte + CaptureDebug bool } // PreparationCallback receives safe preparation provenance before provider work. @@ -90,7 +88,6 @@ type Preparation struct { StartedAt time.Time EndedAt time.Time Duration time.Duration - DataPackagePath string } // PreparationDebug contains content-rich preparation details for an explicitly @@ -127,7 +124,6 @@ type Execution struct { EndedAt time.Time Duration time.Duration Validation Validation - DataPackagePath string RawOutput []byte Debug *ExecutionDebug } diff --git a/internal/promptexec/promptexec_test.go b/internal/promptexec/promptexec_test.go index 523da8c..d13361d 100644 --- a/internal/promptexec/promptexec_test.go +++ b/internal/promptexec/promptexec_test.go @@ -42,7 +42,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq if executor.operationalFailure != nil { return nil, executor.operationalFailure } - preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath} + preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion} var debug *PreparationDebug if request.CaptureDebug { debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}} @@ -57,7 +57,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq if executor.validationRejected { status = ValidationFailed } - result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath, Validation: Validation{Status: status}, RawOutput: []byte("generated content")} + result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")} if request.CaptureDebug { result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")} } @@ -65,7 +65,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq } func TestExecutorLifecycleFixtures(t *testing.T) { - request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", DataPackagePath: "data_package.yaml"} + request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"} t.Run("callback failure prevents provider execution", func(t *testing.T) { executor := &lifecycleExecutor{} callbackError := errors.New("persistence failed") @@ -239,7 +239,6 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) { BackendID: "local", ModelName: "model-name", Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"}, - DataPackagePath: "data-packages/daily/data_package.yaml", } execution := Execution{ RunID: "run-id", @@ -254,7 +253,7 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) { GeneratedHash: "generated-hash", RawOutput: []byte("generated content"), } - text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + preparation.DataPackagePath + execution.RunID + execution.GeneratedHash + text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} { if strings.Contains(text, unwanted) { t.Fatalf("safe values contain %q: %s", unwanted, text) diff --git a/internal/state/filesystem_test.go b/internal/state/filesystem_test.go index f147687..44b4480 100644 --- a/internal/state/filesystem_test.go +++ b/internal/state/filesystem_test.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" @@ -366,21 +367,40 @@ func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) Promp artifact.DataPackagePath = paths.DataPackage artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion artifact.Preparation.PromptHash = "prompt-hash" - artifact.Preparation.DataPackagePath = paths.DataPackage return artifact } +func validPreparationArtifact() PromptPreparationArtifact { + started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + return PromptPreparationArtifact{ + SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded, + ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", + DataPackagePath: "/workspace/data.yaml", Preparation: &promptexec.Preparation{PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0"}, + StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, + } +} + func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact { artifact := validExecutionArtifact() metadata := resolved.Metadata() artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion - artifact.Provenance.DataPackagePath = paths.DataPackage artifact.Paths.RawOutputPath = paths.GeneratedTextRaw return artifact } +func validExecutionArtifact() PromptExecutionArtifact { + started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil) + return PromptExecutionArtifact{ + SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded, + ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", + Provenance: &PromptExecutionProvenance{RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", BackendID: "backend", ModelName: "model", StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second}, + Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, + } +} + func writeJSONFixture(t *testing.T, path string, value any) { t.Helper() data, err := json.Marshal(value) diff --git a/internal/state/prompt_artifacts.go b/internal/state/prompt_artifacts.go index 7bf8928..f81b6c8 100644 --- a/internal/state/prompt_artifacts.go +++ b/internal/state/prompt_artifacts.go @@ -81,7 +81,7 @@ func (a PromptPreparationArtifact) Validate() error { if a.Preparation == nil || a.Error != nil { return fmt.Errorf("successful prompt preparation requires preparation without an error") } - if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath { + if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion { return fmt.Errorf("successful prompt preparation provenance must match the artifact") } case PromptPreparationFailed: @@ -122,7 +122,6 @@ type PromptExecutionProvenance struct { StartedAt time.Time `json:"startedAt"` EndedAt time.Time `json:"endedAt"` Duration time.Duration `json:"duration"` - DataPackagePath string `json:"dataPackagePath"` } // PromptExecutionPaths records only destinations reached by a completed run. @@ -165,7 +164,6 @@ func PromptExecutionProvenanceFrom(value promptexec.Execution) PromptExecutionPr InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage, StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, - DataPackagePath: value.DataPackagePath, } } @@ -255,7 +253,6 @@ func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error { }{ {"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash}, {"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName}, - {"data package path", value.DataPackagePath}, } { if strings.TrimSpace(required.value) == "" { return fmt.Errorf("completed prompt execution provenance %s is required", required.name) diff --git a/internal/state/prompt_artifacts_test.go b/internal/state/prompt_artifacts_test.go deleted file mode 100644 index 1c0babd..0000000 --- a/internal/state/prompt_artifacts_test.go +++ /dev/null @@ -1,207 +0,0 @@ -package state - -import ( - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" - "gitea.maximumdirect.net/eric/weatherreporter/internal/report" -) - -func TestPromptPreparationArtifactValidation(t *testing.T) { - valid := validPreparationArtifact() - if err := valid.Validate(); err != nil { - t.Fatalf("valid successful preparation: %v", err) - } - failed := valid - failed.Status = PromptPreparationFailed - failed.Preparation = nil - failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"} - if err := failed.Validate(); err != nil { - t.Fatalf("valid failed preparation: %v", err) - } - - tests := []struct { - name string - mutate func(*PromptPreparationArtifact) - }{ - {"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }}, - {"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }}, - {"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }}, - {"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }}, - {"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }}, - {"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }}, - {"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }}, - {"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }}, - {"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }}, - {"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }}, - {"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }}, - {"success error", func(a *PromptPreparationArtifact) { - a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"} - }}, - {"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }}, - {"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }}, - {"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - artifact := validPreparationArtifact() - test.mutate(&artifact) - if err := artifact.Validate(); err == nil { - t.Fatalf("Validate() error = nil for %#v", artifact) - } - }) - } -} - -func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) { - tests := []struct { - name string - mutate func(*PromptPreparationArtifact) - }{ - {"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }}, - {"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }}, - {"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }}, - {"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }}, - {"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - artifact := validPreparationArtifact() - artifact.Status = PromptPreparationFailed - artifact.Preparation = nil - artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"} - test.mutate(&artifact) - if err := artifact.Validate(); err == nil { - t.Fatalf("Validate() error = nil for %#v", artifact) - } - }) - } -} - -func TestPromptExecutionArtifactValidation(t *testing.T) { - valid := validExecutionArtifact() - valid.Provenance.RunID = "provider-run-different-from-weatherreporter" - valid.Provenance.GeneratedHash = "" - valid.Provenance.Usage = promptexec.TokenUsage{} - if err := valid.Validate(); err != nil { - t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err) - } - rejected := validExecutionArtifact() - rejected.Status = PromptExecutionValidationRejected - validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"}) - rejected.Validation = &validation - if err := rejected.Validate(); err != nil { - t.Fatalf("valid validation rejection: %v", err) - } - failed := validFailedExecutionArtifact() - if err := failed.Validate(); err != nil { - t.Fatalf("valid operational failure: %v", err) - } - - tests := []struct { - name string - mutate func(*PromptExecutionArtifact) - }{ - {"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }}, - {"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }}, - {"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }}, - {"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }}, - {"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }}, - {"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }}, - {"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }}, - {"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }}, - {"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }}, - {"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }}, - {"wrong validation", func(a *PromptExecutionArtifact) { - value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil) - a.Validation = &value - }}, - {"operational error", func(a *PromptExecutionArtifact) { - a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"} - }}, - {"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }}, - {"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }}, - {"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }}, - {"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }}, - {"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }}, - {"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }}, - {"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }}, - {"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }}, - {"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }}, - {"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }}, - {"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }}, - {"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }}, - {"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - artifact := validExecutionArtifact() - test.mutate(&artifact) - if err := artifact.Validate(); err == nil { - t.Fatalf("Validate() error = nil for %#v", artifact) - } - }) - } -} - -func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) { - tests := []struct { - name string - mutate func(*PromptExecutionArtifact) - }{ - {"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }}, - {"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }}, - {"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }}, - {"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - artifact := validFailedExecutionArtifact() - test.mutate(&artifact) - if err := artifact.Validate(); err == nil { - t.Fatalf("Validate() error = nil for %#v", artifact) - } - }) - } -} - -func validPreparationArtifact() PromptPreparationArtifact { - started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - return PromptPreparationArtifact{ - SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded, - ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", - PromptVersion: "2.0.0", DataPackagePath: "/workspace/data.yaml", - Preparation: &promptexec.Preparation{ - PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", - DataPackagePath: "/workspace/data.yaml", - }, - StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, - } -} - -func validExecutionArtifact() PromptExecutionArtifact { - started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil) - return PromptExecutionArtifact{ - SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded, - ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", - Provenance: &PromptExecutionProvenance{ - RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", - PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", - BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml", - StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, - }, - Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, - } -} - -func validFailedExecutionArtifact() PromptExecutionArtifact { - started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - return PromptExecutionArtifact{ - SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed, - ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", - StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}, - } -}