eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete comparison command output

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-02 06:02:24 +00:00
parent b489c56a48
commit ccf6b66880
4 changed files with 355 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
internal/cli/result_test.go
View File

@@ -1,12 +1,15 @@
package cli
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
@@ -29,3 +32,106 @@ func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
}
}
}
func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
result := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
Total: 2, Succeeded: 1, Failed: 1,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
},
}
summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles"))
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatal(err)
}
previous := -1
for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} {
position := strings.Index(string(data), `"`+field+`":`)
if field == "error" {
position = strings.LastIndex(string(data), `"`+field+`":`)
}
if position <= previous {
t.Fatalf("field order for %q in %s", field, data)
}
previous = position
}
for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} {
if strings.Contains(string(data), unsafe) {
t.Fatalf("summary includes unsafe content %q: %s", unsafe, data)
}
}
}
func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) {
result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}}
summary := newComparisonSummary(result, context.Canceled)
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatal(err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
t.Fatal(err)
}
for _, omitted := range []string{"manifestPath", "dataPackagePath"} {
if _, exists := fields[omitted]; exists {
t.Fatalf("summary includes unpublished %s: %s", omitted, data)
}
}
publicationResult := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded},
},
}
unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material")
publicationSummary := newComparisonSummary(publicationResult, unsafe)
if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" {
safe := publicationSummary.Error
t.Fatalf("safe error = %#v", safe)
}
}
func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
failure := comparison.NewSafeError("generation", "execute prompt failed")
complete := &app.ComparisonResult{
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
Total: 2, Succeeded: 2,
Results: []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"},
},
}
if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil {
t.Fatalf("complete summary = %#v", summary)
}
allFailed := *complete
allFailed.Succeeded, allFailed.Failed = 0, 2
allFailed.Results = []app.ComparisonProfileResult{
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure},
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure},
}
summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles"))
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil {
t.Fatalf("all-failed summary = %#v", summary)
}
}