package cli import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "reflect" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/app" "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" ) func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) { opts, err := parseComparisonFlags(app.ReportDaily, []string{ "--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep", "--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet", }) if err != nil { t.Fatalf("parseComparisonFlags() error = %v", err) } wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"} if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet { t.Fatalf("options = %#v", opts) } } func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) { workingDir := t.TempDir() configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n units: metric\n timezone: America/Chicago\noutput:\n directory: configured-reports\npromptkit:\n profile: configured-default\n") clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)} var factoryConfig PromptExecutorConfig factoryCalls := 0 executor := &factoryExecutor{} runner := Runner{ Clock: clock, WorkingDir: workingDir, ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) { factoryCalls++ factoryConfig = value return executor, nil }, } req, opts, err := runner.resolveComparisonAction([]string{ "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath, }) if err != nil { t.Fatalf("resolveComparisonAction() error = %v", err) } if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" { t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls) } if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) { t.Fatalf("profile IDs = %#v, want %#v", got, want) } if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet { t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig) } location, loadErr := time.LoadLocation("America/New_York") if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" { t.Fatalf("date/location = %v/%v", req.Date, loadErr) } } func TestResolveComparisonActionMatchesReportDatePolicies(t *testing.T) { configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n timezone: America/Chicago\n") runner := comparisonRunner(t, t.TempDir()) for _, test := range []struct { name string args []string wantDay string wantErr bool }{ {name: "daily requires date", args: []string{"daily", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true}, {name: "today uses current local date", args: []string{"today", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-29"}, {name: "today accepts date", args: []string{"today", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-30"}, {name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true}, {name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true}, } { t.Run(test.name, func(t *testing.T) { req, _, err := runner.resolveComparisonAction(test.args) if test.wantErr { if err == nil { t.Fatal("resolveComparisonAction() error = nil") } return } if err != nil || req.Date.Format("2006-01-02") != test.wantDay { t.Fatalf("request/error = %#v/%v", req, err) } }) } } func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) { workingDir := t.TempDir() configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured/../reports\n") req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{ "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath, }) if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" { t.Fatalf("request/error = %#v/%v", req, err) } } func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) { for _, test := range []struct { name string args []string }{ {name: "missing report", args: nil}, {name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}}, {name: "single profile", args: []string{"today", "--profile", "one"}}, {name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}}, {name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}}, {name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}}, {name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}}, {name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}}, } { t.Run(test.name, func(t *testing.T) { factoryCalls, applicationCalls := 0, 0 runner := Runner{ Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { factoryCalls++ return &factoryExecutor{}, nil }, compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) { applicationCalls++ return nil, nil }, } if _, err := runner.executeComparison(context.Background(), test.args); err == nil { t.Fatal("executeComparison() error = nil") } if factoryCalls != 0 || applicationCalls != 0 { t.Fatalf("factory/application calls = %d/%d", factoryCalls, applicationCalls) } }) } } func TestExecuteComparisonUsesOneExecutorAndInjectedApplication(t *testing.T) { workingDir := t.TempDir() configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n") executor := &factoryExecutor{} factoryCalls, applicationCalls := 0, 0 var received app.ComparisonRequest wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"} runner := Runner{ Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir, ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { factoryCalls++ return executor, nil }, compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) { applicationCalls++ received = req return wantResult, errors.New("comparison completed with 1 failed profiles") }, } result, err := runner.executeComparison(context.Background(), []string{ "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath, }) if result != wantResult || err == nil || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace { t.Fatalf("result/error/calls/request = %#v/%v/%d/%d/%#v", result, err, factoryCalls, applicationCalls, received) } if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) { t.Fatalf("profile IDs = %#v", received.ProfileIDs) } } func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) { workingDir := t.TempDir() configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n") profileFailure := comparison.NewSafeError("generation", "execute prompt failed") result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"}, {Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure}, }) partialErr := errors.New("comparison completed with 1 failed profiles") factoryCalls, applicationCalls := 0, 0 runner := Runner{ Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir, ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { factoryCalls++ return &factoryExecutor{}, nil }, compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) { applicationCalls++ return result, partialErr }, } var stdout, stderr bytes.Buffer err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr) if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 { t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String()) } var summary comparisonSummary if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil { t.Fatalf("decode summary: %v\n%s", err, stdout.String()) } if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil { t.Fatalf("summary = %#v", summary) } } func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) { workingDir := t.TempDir() configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n") for _, test := range []struct { name string outputArgument []string outputDirectory string wantRequestPath string }{ {name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")}, {name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")}, } { t.Run(test.name, func(t *testing.T) { var received app.ComparisonRequest runner := comparisonRunner(t, workingDir) runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) { received = req return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{ {Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")}, {Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")}, }), nil } args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath} args = append(args, test.outputArgument...) var stdout, stderr bytes.Buffer if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil { t.Fatalf("Run() error = %v", err) } var summary comparisonSummary if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath { t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received) } }) } } func TestCompareCommandQuietPreservesFailure(t *testing.T) { configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n") failure := errors.New("comparison completed with 2 failed profiles") calls := 0 runner := comparisonRunner(t, t.TempDir()) runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) { calls++ return comparisonResult("/reports/comparison-daily", nil), failure } var stdout, stderr bytes.Buffer err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr) if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 { t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String()) } } func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) { runner := comparisonRunner(t, t.TempDir()) called := false runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) { called = true return nil, nil } var stdout, stderr bytes.Buffer err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr) if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 { t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String()) } } func TestCompareHelpIncludesCommand(t *testing.T) { var stdout, stderr bytes.Buffer if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") { t.Fatalf("help/error = %q/%v", stdout.String(), err) } } func comparisonRunner(t *testing.T, workingDir string) Runner { t.Helper() return Runner{ Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir, ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil }, } } func comparisonConfigPath(t *testing.T, contents string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { t.Fatal(err) } return path } func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult { started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) result := &app.ComparisonResult{ ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64), StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory, ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"), Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results), } for _, profile := range results { if profile.Status == "succeeded" { result.Succeeded++ } else { result.Failed++ } } return result }