package cli import ( "context" "encoding/json" "errors" "fmt" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/app" "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) func TestGenerateSummaryUsesActiveResultFields(t *testing.T) { generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil) if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) || len(summary.SourceWarnings) != 1 { t.Fatalf("summary = %#v", summary) } data, err := json.Marshal(summary) if err != nil { t.Fatalf("Marshal() error = %v", err) } for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} { if strings.Contains(string(data), forbidden) { t.Fatalf("summary includes %q: %s", forbidden, data) } } } func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) { started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) profileFailure := comparison.NewSafeError("generation", "execute prompt failed") result := &app.ComparisonResult{ ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64), StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml", Total: 2, Succeeded: 1, Failed: 1, Results: []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"}, {Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure}, }, } summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles")) if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil { t.Fatalf("summary = %#v", summary) } data, err := json.Marshal(summary) if err != nil { t.Fatal(err) } previous := -1 for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} { position := strings.Index(string(data), `"`+field+`":`) if field == "error" { position = strings.LastIndex(string(data), `"`+field+`":`) } if position <= previous { t.Fatalf("field order for %q in %s", field, data) } previous = position } for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} { if strings.Contains(string(data), unsafe) { t.Fatalf("summary includes unsafe content %q: %s", unsafe, data) } } } func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) { result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}} summary := newComparisonSummary(result, context.Canceled) if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil { t.Fatalf("summary = %#v", summary) } data, err := json.Marshal(summary) if err != nil { t.Fatal(err) } var fields map[string]json.RawMessage if err := json.Unmarshal(data, &fields); err != nil { t.Fatal(err) } for _, omitted := range []string{"manifestPath", "dataPackagePath"} { if _, exists := fields[omitted]; exists { t.Fatalf("summary includes unpublished %s: %s", omitted, data) } } publicationResult := &app.ComparisonResult{ ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2, Results: []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded}, {Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded}, }, } unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material") publicationSummary := newComparisonSummary(publicationResult, unsafe) if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" { safe := publicationSummary.Error t.Fatalf("safe error = %#v", safe) } } func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) { started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) failure := comparison.NewSafeError("generation", "execute prompt failed") complete := &app.ComparisonResult{ ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64), StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)}, OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml", Total: 2, Succeeded: 2, Results: []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"}, {Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"}, }, } if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil { t.Fatalf("complete summary = %#v", summary) } allFailed := *complete allFailed.Succeeded, allFailed.Failed = 0, 2 allFailed.Results = []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure}, {Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure}, } summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles")) if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil { t.Fatalf("all-failed summary = %#v", summary) } } func TestSafeComparisonSummaryErrorClassifiesWrappedFailures(t *testing.T) { unsafeDetail := "unsafe filesystem and provider detail" unsafeCause := errors.New(unsafeDetail) for _, test := range []struct { name string err error category string message string }{ { name: "aggregate profile failure", err: fmt.Errorf("outer wrapper: %w", errors.New("comparison completed with 2 failed profiles")), category: "application", message: "comparison completed with 2 failed profiles", }, { name: "canceled", err: fmt.Errorf("outer wrapper: %w", context.Canceled), category: "canceled", message: "comparison canceled", }, { name: "deadline exceeded", err: fmt.Errorf("outer wrapper: %w", context.DeadlineExceeded), category: "deadline_exceeded", message: "comparison deadline exceeded", }, { name: "prompt operation", err: fmt.Errorf("outer wrapper: %w", promptexec.NewError(promptexec.Generation, "unsafe prompt detail", fmt.Errorf("%w: %s", context.Canceled, unsafeDetail))), category: string(promptexec.Generation), message: "comparison prompt operation failed", }, { name: "destination preflight", err: fmt.Errorf("outer wrapper: %w", &comparison.DestinationError{ Kind: comparison.DestinationNotEmpty, Target: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail), }), category: "destination_not_empty", message: "comparison destination preflight failed", }, { name: "complete cleanup recovery", err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{ RecoveryState: comparison.BackupRecoveryComplete, RecoveryPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail), }), category: "publication_cleanup", message: "comparison published but a complete prior bundle remains", }, { name: "partial cleanup remnants", err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{ RecoveryState: comparison.BackupRecoveryPartial, RecoveryPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail), }), category: "publication_cleanup", message: "comparison published but partial cleanup remnants remain", }, { name: "absent cleanup recovery", err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{ RecoveryState: comparison.BackupRecoveryAbsent, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail), }), category: "publication_cleanup", message: "comparison published but no prior bundle remains", }, { name: "unknown cleanup recovery", err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{ RecoveryState: comparison.BackupRecoveryUnknown, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail), }), category: "publication_cleanup", message: "comparison published but cleanup recovery state is unknown", }, { name: "unknown", err: fmt.Errorf("outer wrapper: %w", unsafeCause), category: "application", message: "comparison did not complete", }, } { t.Run(test.name, func(t *testing.T) { result := &app.ComparisonResult{ComparisonID: "comparison_test", ReportID: report.Daily} summary := newComparisonSummary(result, test.err) if summary.Error == nil || summary.Error.Category != test.category || summary.Error.Message != test.message { t.Fatalf("summary error = %#v, want %q/%q", summary.Error, test.category, test.message) } data, err := json.Marshal(summary) if err != nil { t.Fatal(err) } if strings.Contains(string(data), unsafeDetail) { t.Fatalf("summary includes unsafe detail: %s", data) } }) } }