Complete comparison command output

internal/cli/result.go

@@ -1,9 +1,15 @@
 package cli
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
 	"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
+	"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
 	"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
 	"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
 	"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
@@ -12,6 +18,7 @@ import (
 const (
 	commandGenerate = "generate"
 	commandRun      = "run"
+	commandCompare  = "compare"
 
 	summaryStatusSucceeded = "succeeded"
 	summaryStatusFailed    = "failed"
@@ -67,6 +74,41 @@ type batchSummary struct {
 	Error        string                       `json:"error,omitempty"`
 }
 
+type comparisonSummary struct {
+	Command         string                     `json:"command"`
+	ComparisonID    string                     `json:"comparisonId"`
+	ReportID        report.ID                  `json:"reportId"`
+	ReportName      string                     `json:"reportName"`
+	PromptID        string                     `json:"promptId"`
+	PromptVersion   string                     `json:"promptVersion"`
+	PromptHash      string                     `json:"promptHash"`
+	Status          string                     `json:"status"`
+	StartedAt       time.Time                  `json:"startedAt"`
+	FinishedAt      time.Time                  `json:"finishedAt"`
+	Timezone        string                     `json:"timezone"`
+	ValidPeriod     timeutil.Period            `json:"validPeriod"`
+	OutputDirectory string                     `json:"outputDirectory"`
+	ManifestPath    string                     `json:"manifestPath,omitempty"`
+	DataPackagePath string                     `json:"dataPackagePath,omitempty"`
+	Total           int                        `json:"total"`
+	Succeeded       int                        `json:"succeeded"`
+	Failed          int                        `json:"failed"`
+	Results         []comparisonProfileSummary `json:"results"`
+	Error           *comparison.SafeError      `json:"error,omitempty"`
+}
+
+type comparisonProfileSummary struct {
+	Position         int                   `json:"position"`
+	ProfileID        string                `json:"profileId"`
+	BackendID        string                `json:"backendId,omitempty"`
+	ModelName        string                `json:"modelName"`
+	Status           string                `json:"status"`
+	ValidationStatus string                `json:"validationStatus,omitempty"`
+	ReportPath       string                `json:"reportPath,omitempty"`
+	LLMDebugPath     string                `json:"llmDebugPath,omitempty"`
+	Error            *comparison.SafeError `json:"error,omitempty"`
+}
+
 func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
 	summary := generateSummary{Command: commandGenerate}
 	if result == nil {
@@ -139,6 +181,66 @@ func newBatchSummary(result *app.BatchResult) batchSummary {
 	return summary
 }
 
+func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSummary {
+	summary := comparisonSummary{Command: commandCompare, Results: []comparisonProfileSummary{}}
+	if result == nil {
+		return summary
+	}
+	summary.ComparisonID = result.ComparisonID
+	summary.ReportID, summary.ReportName = result.ReportID, result.ReportName
+	summary.PromptID, summary.PromptVersion, summary.PromptHash = result.PromptID, result.PromptVersion, result.PromptHash
+	summary.StartedAt, summary.FinishedAt = result.StartedAt, result.FinishedAt
+	summary.Timezone, summary.ValidPeriod = result.Timezone, result.ValidPeriod
+	summary.OutputDirectory = result.OutputDirectory
+	summary.ManifestPath, summary.DataPackagePath = result.ManifestPath, result.DataPackagePath
+	summary.Total, summary.Succeeded, summary.Failed = result.Total, result.Succeeded, result.Failed
+	for _, profile := range result.Results {
+		summary.Results = append(summary.Results, comparisonProfileSummary{
+			Position: profile.Position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
+			Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
+			LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
+		})
+	}
+	summary.Status = comparisonSummaryStatus(result, err)
+	if err != nil {
+		summary.Error = safeComparisonSummaryError(err)
+	}
+	return summary
+}
+
+func comparisonSummaryStatus(result *app.ComparisonResult, err error) string {
+	if result == nil || err != nil || result.Total < 2 || result.Succeeded != result.Total || result.Failed != 0 || result.ManifestPath == "" || result.DataPackagePath == "" {
+		return summaryStatusFailed
+	}
+	return summaryStatusSucceeded
+}
+
+func safeComparisonSummaryError(err error) *comparison.SafeError {
+	message := "comparison did not complete"
+	if aggregate, ok := comparisonAggregateErrorMessage(err.Error()); ok {
+		message = aggregate
+	} else if errors.Is(err, context.DeadlineExceeded) {
+		message = "comparison deadline exceeded"
+	} else if errors.Is(err, context.Canceled) {
+		message = "comparison canceled"
+	}
+	safe := comparison.NewSafeError("application", message)
+	return &safe
+}
+
+func comparisonAggregateErrorMessage(value string) (string, bool) {
+	const prefix = "comparison completed with "
+	const suffix = " failed profiles"
+	if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
+		return "", false
+	}
+	count, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix))
+	if err != nil || count < 1 {
+		return "", false
+	}
+	return fmt.Sprintf("comparison completed with %d failed profiles", count), true
+}
+
 func batchSummaryStatus(result *app.BatchResult) string {
 	if result == nil {
 		return ""