369 lines
19 KiB
Go
369 lines
19 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
)
|
|
|
|
func TestParseComparisonFlagsPreservesProfileOrder(t *testing.T) {
|
|
opts, err := parseComparisonFlags(app.ReportDaily, []string{
|
|
"--profile", "weather-light", "--profile=weather-balanced", "--profile", "weather-deep",
|
|
"--date", "2026-05-29", "--out-dir", "reports", "--replace", "--quiet",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("parseComparisonFlags() error = %v", err)
|
|
}
|
|
wantProfiles := []string{"weather-light", "weather-balanced", "weather-deep"}
|
|
if !reflect.DeepEqual([]string(opts.ProfileIDs), wantProfiles) || opts.Date != "2026-05-29" || opts.OutputDir != "reports" || !opts.Replace || !opts.Quiet {
|
|
t.Fatalf("options = %#v", opts)
|
|
}
|
|
}
|
|
|
|
func TestResolveComparisonActionBuildsExplicitRequest(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n units: metric\n timezone: America/Chicago\noutput:\n directory: configured-reports\npromptkit:\n profile: configured-default\n")
|
|
clock := timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}
|
|
var factoryConfig PromptExecutorConfig
|
|
factoryCalls := 0
|
|
executor := &factoryExecutor{}
|
|
runner := Runner{
|
|
Clock: clock, WorkingDir: workingDir,
|
|
ExecutorFactory: func(value PromptExecutorConfig) (promptexec.Executor, error) {
|
|
factoryCalls++
|
|
factoryConfig = value
|
|
return executor, nil
|
|
},
|
|
}
|
|
req, opts, err := runner.resolveComparisonAction([]string{
|
|
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep",
|
|
"--out-dir", "bundles/../comparison", "--replace", "--llm-debug-dir", "/tmp/debug", "--units", "imperial", "--tz", "America/New_York", "--config", configPath,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("resolveComparisonAction() error = %v", err)
|
|
}
|
|
if factoryCalls != 1 || req.Executor != executor || req.Clock != clock || req.WorkingDir != workingDir || req.OutputDir != filepath.Join(workingDir, "comparison") || !req.Replace || req.LLMDebugDir != "/tmp/debug" {
|
|
t.Fatalf("request/factory calls = %#v/%#v/%d", req, factoryConfig, factoryCalls)
|
|
}
|
|
if got, want := req.ProfileIDs, []string{"weather-light", "weather-deep"}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("profile IDs = %#v, want %#v", got, want)
|
|
}
|
|
if req.Config.WeatherAPI.Units != "imperial" || req.Config.WeatherAPI.Timezone != "America/New_York" || req.Config.Output.Directory != "configured-reports" || factoryConfig.Profile != "" || opts.Quiet {
|
|
t.Fatalf("configuration/options/factory config = %#v/%#v/%#v", req.Config, opts, factoryConfig)
|
|
}
|
|
location, loadErr := time.LoadLocation("America/New_York")
|
|
if loadErr != nil || req.Date.Location().String() != location.String() || req.Date.Format("2006-01-02") != "2026-05-29" {
|
|
t.Fatalf("date/location = %v/%v", req.Date, loadErr)
|
|
}
|
|
}
|
|
|
|
func TestResolveComparisonActionMatchesReportDatePolicies(t *testing.T) {
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n timezone: America/Chicago\n")
|
|
runner := comparisonRunner(t, t.TempDir())
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
wantDay string
|
|
wantErr bool
|
|
}{
|
|
{name: "daily requires date", args: []string{"daily", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
|
{name: "today uses current local date", args: []string{"today", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-29"},
|
|
{name: "today accepts date", args: []string{"today", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantDay: "2026-05-30"},
|
|
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
|
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-30", "--profile", "one", "--profile", "two", "--config", configPath}, wantErr: true},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, _, err := runner.resolveComparisonAction(test.args)
|
|
if test.wantErr {
|
|
if err == nil {
|
|
t.Fatal("resolveComparisonAction() error = nil")
|
|
}
|
|
return
|
|
}
|
|
if err != nil || req.Date.Format("2006-01-02") != test.wantDay {
|
|
t.Fatalf("request/error = %#v/%v", req, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResolveComparisonActionLeavesConfiguredOutputWithoutOverride(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured/../reports\n")
|
|
req, _, err := comparisonRunner(t, workingDir).resolveComparisonAction([]string{
|
|
"daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath,
|
|
})
|
|
if err != nil || req.OutputDir != "" || req.Config.Output.Directory != "configured/../reports" {
|
|
t.Fatalf("request/error = %#v/%v", req, err)
|
|
}
|
|
}
|
|
|
|
func TestComparisonInputFailuresDoNotConstructOrExecute(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
}{
|
|
{name: "missing report", args: nil},
|
|
{name: "unknown report", args: []string{"unknown", "--profile", "one", "--profile", "two"}},
|
|
{name: "single profile", args: []string{"today", "--profile", "one"}},
|
|
{name: "blank profile", args: []string{"today", "--profile", "one", "--profile", " \t"}},
|
|
{name: "duplicate profile", args: []string{"today", "--profile", "one", "--profile", "one"}},
|
|
{name: "unexpected argument", args: []string{"today", "extra", "--profile", "one", "--profile", "two"}},
|
|
{name: "unsupported output flag", args: []string{"today", "--out", "report.md", "--profile", "one", "--profile", "two"}},
|
|
{name: "configuration failure", args: []string{"today", "--profile", "one", "--profile", "two", "--config", filepath.Join(t.TempDir(), "missing.yml")}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
factoryCalls, applicationCalls := 0, 0
|
|
runner := Runner{
|
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)},
|
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
|
factoryCalls++
|
|
return &factoryExecutor{}, nil
|
|
},
|
|
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
applicationCalls++
|
|
return nil, nil
|
|
},
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
if err := runner.Run(context.Background(), append([]string{"compare"}, test.args...), &stdout, &stderr); err == nil {
|
|
t.Fatal("Run() error = nil")
|
|
}
|
|
if factoryCalls != 0 || applicationCalls != 0 || stdout.Len() != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("factory/application calls/output = %d/%d/%q/%q", factoryCalls, applicationCalls, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandUsesOneExecutorAndInjectedApplication(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
|
executor := &factoryExecutor{}
|
|
factoryCalls, applicationCalls := 0, 0
|
|
var received app.ComparisonRequest
|
|
wantResult := &app.ComparisonResult{ComparisonID: "comparison_test"}
|
|
comparisonErr := errors.New("comparison completed with 1 failed profiles")
|
|
runner := Runner{
|
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
|
factoryCalls++
|
|
return executor, nil
|
|
},
|
|
compareDetailed: func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
applicationCalls++
|
|
received = req
|
|
return wantResult, comparisonErr
|
|
},
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
err := runner.Run(context.Background(), []string{
|
|
"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--out-dir", "comparison", "--replace", "--config", configPath,
|
|
}, &stdout, &stderr)
|
|
if !errors.Is(err, comparisonErr) || factoryCalls != 1 || applicationCalls != 1 || received.Executor != executor || received.OutputDir != filepath.Join(workingDir, "comparison") || !received.Replace || stdout.Len() == 0 || stderr.Len() != 0 {
|
|
t.Fatalf("error/calls/request/output = %v/%d/%d/%#v/%q/%q", err, factoryCalls, applicationCalls, received, stdout.String(), stderr.String())
|
|
}
|
|
if !reflect.DeepEqual(received.ProfileIDs, []string{"weather-light", "weather-deep"}) {
|
|
t.Fatalf("profile IDs = %#v", received.ProfileIDs)
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
|
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
|
result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
|
|
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
|
})
|
|
partialErr := errors.New("comparison completed with 1 failed profiles")
|
|
factoryCalls, applicationCalls := 0, 0
|
|
runner := Runner{
|
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
|
factoryCalls++
|
|
return &factoryExecutor{}, nil
|
|
},
|
|
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
applicationCalls++
|
|
return result, partialErr
|
|
},
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
|
if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
|
|
t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
|
|
}
|
|
var summary comparisonSummary
|
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
|
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
|
}
|
|
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandReportsCommittedBundleWhenCleanupFails(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
|
outputDirectory := filepath.Join(workingDir, "comparison-daily")
|
|
result := comparisonResult(outputDirectory, []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "01-weather-light.md")},
|
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(outputDirectory, "02-weather-deep.md")},
|
|
})
|
|
backupPath := filepath.Join(workingDir, ".comparison-daily.backup-retained")
|
|
cleanupCause := errors.New("filesystem cleanup detail")
|
|
cleanupErr := &comparison.PublicationCleanupError{RetainedBackupPath: backupPath, Err: cleanupCause}
|
|
runner := comparisonRunner(t, workingDir)
|
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
return result, cleanupErr
|
|
}
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
|
if !errors.Is(err, cleanupCause) || stderr.Len() != 0 {
|
|
t.Fatalf("Run() error/stderr = %v/%q", err, stderr.String())
|
|
}
|
|
var summary comparisonSummary
|
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
|
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
|
}
|
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Category != "publication_cleanup" || summary.Error.Message != "comparison published but cleanup did not complete" || summary.ManifestPath != result.ManifestPath || summary.DataPackagePath != result.DataPackagePath || summary.Results[0].ReportPath == "" {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
for _, unsafe := range []string{cleanupCause.Error(), backupPath} {
|
|
if strings.Contains(stdout.String(), unsafe) {
|
|
t.Fatalf("summary contains unsafe recovery detail %q: %s", unsafe, stdout.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n")
|
|
for _, test := range []struct {
|
|
name string
|
|
outputArgument []string
|
|
outputDirectory string
|
|
wantRequestPath string
|
|
}{
|
|
{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
|
|
{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var received app.ComparisonRequest
|
|
runner := comparisonRunner(t, workingDir)
|
|
runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
received = req
|
|
return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
|
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
|
|
}), nil
|
|
}
|
|
args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
|
|
args = append(args, test.outputArgument...)
|
|
var stdout, stderr bytes.Buffer
|
|
if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
|
|
t.Fatalf("Run() error = %v", err)
|
|
}
|
|
var summary comparisonSummary
|
|
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
|
|
t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandQuietPreservesFailure(t *testing.T) {
|
|
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
|
failure := errors.New("comparison completed with 2 failed profiles")
|
|
calls := 0
|
|
runner := comparisonRunner(t, t.TempDir())
|
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
calls++
|
|
return comparisonResult("/reports/comparison-daily", nil), failure
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
|
|
if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
|
|
runner := comparisonRunner(t, t.TempDir())
|
|
called := false
|
|
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
|
called = true
|
|
return nil, nil
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
|
|
if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestCompareHelpIncludesCommand(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") || !strings.Contains(stdout.String(), "--date YYYY-MM-DD") || !strings.Contains(stdout.String(), "Suppress action summaries and routine batch status output.") {
|
|
t.Fatalf("help/error = %q/%v", stdout.String(), err)
|
|
}
|
|
}
|
|
|
|
func TestCompareHelpDoesNotRequireConfiguration(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if err := (Runner{}).Run(context.Background(), []string{"compare", "--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || stderr.Len() != 0 {
|
|
t.Fatalf("help/error/stderr = %q/%v/%q", stdout.String(), err, stderr.String())
|
|
}
|
|
}
|
|
|
|
func comparisonRunner(t *testing.T, workingDir string) Runner {
|
|
t.Helper()
|
|
return Runner{
|
|
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
|
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) { return &factoryExecutor{}, nil },
|
|
}
|
|
}
|
|
|
|
func comparisonConfigPath(t *testing.T, contents string) string {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "config.yml")
|
|
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
|
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
result := &app.ComparisonResult{
|
|
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
|
|
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
|
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
|
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
|
|
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
|
|
Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
|
|
}
|
|
for _, profile := range results {
|
|
if profile.Status == "succeeded" {
|
|
result.Succeeded++
|
|
} else {
|
|
result.Failed++
|
|
}
|
|
}
|
|
return result
|
|
}
|