Complete comparison command output
This commit is contained in:
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
@@ -175,13 +177,110 @@ func TestExecuteComparisonUsesOneExecutorAndInjectedApplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareRemainsAbsentFromRootDispatchAndHelp(t *testing.T) {
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := runner.Run(context.Background(), []string{"compare", "today"}, &stdout, &stderr); err == nil || err.Error() != `unknown command "compare"` {
|
||||
t.Fatalf("Run(compare) error = %v", err)
|
||||
func TestCompareCommandWritesStructuredPartialFailure(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
||||
result := comparisonResult("/reports/comparison-daily", []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md", LLMDebugPath: "/debug/comparison-light"},
|
||||
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
||||
})
|
||||
partialErr := errors.New("comparison completed with 1 failed profiles")
|
||||
factoryCalls, applicationCalls := 0, 0
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 30, 0, 0, time.UTC)}, WorkingDir: workingDir,
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
factoryCalls++
|
||||
return &factoryExecutor{}, nil
|
||||
},
|
||||
compareDetailed: func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
applicationCalls++
|
||||
return result, partialErr
|
||||
},
|
||||
}
|
||||
if err := runner.Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || strings.Contains(stdout.String(), "compare") {
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}, &stdout, &stderr)
|
||||
if !errors.Is(err, partialErr) || factoryCalls != 1 || applicationCalls != 1 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/calls/stderr = %v/%d/%d/%q", err, factoryCalls, applicationCalls, stderr.String())
|
||||
}
|
||||
var summary comparisonSummary
|
||||
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil {
|
||||
t.Fatalf("decode summary: %v\n%s", err, stdout.String())
|
||||
}
|
||||
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != partialErr.Error() || summary.OutputDirectory != result.OutputDirectory || len(summary.Results) != 2 || summary.Results[0].ReportPath != result.Results[0].ReportPath || summary.Results[0].LLMDebugPath != result.Results[0].LLMDebugPath || summary.Results[1].Error == nil {
|
||||
t.Fatalf("summary = %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandWritesSuccessForDefaultAndExplicitDestinations(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\noutput:\n directory: configured-reports\n")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
outputArgument []string
|
||||
outputDirectory string
|
||||
wantRequestPath string
|
||||
}{
|
||||
{name: "configured default", outputDirectory: filepath.Join(workingDir, "configured-reports", "comparison-daily-2026-05-29")},
|
||||
{name: "explicit directory", outputArgument: []string{"--out-dir", "published"}, outputDirectory: filepath.Join(workingDir, "published"), wantRequestPath: filepath.Join(workingDir, "published")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var received app.ComparisonRequest
|
||||
runner := comparisonRunner(t, workingDir)
|
||||
runner.compareDetailed = func(_ context.Context, req app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
received = req
|
||||
return comparisonResult(test.outputDirectory, []app.ComparisonProfileResult{
|
||||
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "01-weather-light.md")},
|
||||
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: filepath.Join(test.outputDirectory, "02-weather-deep.md")},
|
||||
}), nil
|
||||
}
|
||||
args := []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--config", configPath}
|
||||
args = append(args, test.outputArgument...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := runner.Run(context.Background(), args, &stdout, &stderr); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
var summary comparisonSummary
|
||||
if err := json.Unmarshal(stdout.Bytes(), &summary); err != nil || summary.Status != summaryStatusSucceeded || summary.OutputDirectory != test.outputDirectory || stderr.Len() != 0 || received.OutputDir != test.wantRequestPath {
|
||||
t.Fatalf("summary/error/stderr/request = %#v/%v/%q/%#v", summary, err, stderr.String(), received)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandQuietPreservesFailure(t *testing.T) {
|
||||
configPath := comparisonConfigPath(t, "weather_api:\n base_url: https://weather.api.example.com/\n")
|
||||
failure := errors.New("comparison completed with 2 failed profiles")
|
||||
calls := 0
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
calls++
|
||||
return comparisonResult("/reports/comparison-daily", nil), failure
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "daily", "--date", "2026-05-29", "--profile", "weather-light", "--profile", "weather-deep", "--quiet", "--config", configPath}, &stdout, &stderr)
|
||||
if !errors.Is(err, failure) || calls != 1 || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/calls/stdout/stderr = %v/%d/%q/%q", err, calls, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareCommandLeavesPreExecutionFailuresUnstructured(t *testing.T) {
|
||||
runner := comparisonRunner(t, t.TempDir())
|
||||
called := false
|
||||
runner.compareDetailed = func(context.Context, app.ComparisonRequest) (*app.ComparisonResult, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := runner.Run(context.Background(), []string{"compare", "today", "--profile", "only-one"}, &stdout, &stderr)
|
||||
if err == nil || called || stdout.Len() != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("error/called/stdout/stderr = %v/%t/%q/%q", err, called, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareHelpIncludesCommand(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := (Runner{}).Run(context.Background(), []string{"--help"}, &stdout, &stderr); err != nil || !strings.Contains(stdout.String(), "weatherreporter compare REPORT") || !strings.Contains(stdout.String(), "--profile PROFILE") {
|
||||
t.Fatalf("help/error = %q/%v", stdout.String(), err)
|
||||
}
|
||||
}
|
||||
@@ -202,3 +301,23 @@ func comparisonConfigPath(t *testing.T, contents string) string {
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func comparisonResult(outputDirectory string, results []app.ComparisonProfileResult) *app.ComparisonResult {
|
||||
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||
result := &app.ComparisonResult{
|
||||
ComparisonID: "comparison_run-123", ReportID: "daily", ReportName: "Daily Report",
|
||||
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
||||
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
||||
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: outputDirectory,
|
||||
ManifestPath: filepath.Join(outputDirectory, "comparison.json"), DataPackagePath: filepath.Join(outputDirectory, "data-package.yml"),
|
||||
Results: append([]app.ComparisonProfileResult(nil), results...), Total: len(results),
|
||||
}
|
||||
for _, profile := range results {
|
||||
if profile.Status == "succeeded" {
|
||||
result.Succeeded++
|
||||
} else {
|
||||
result.Failed++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user