236 lines
12 KiB
Go
236 lines
12 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
|
|
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", SourceWarnings: []weatherdata.SourceWarning{{Source: "alerts", Message: "source unavailable"}}, ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
|
|
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) || len(summary.SourceWarnings) != 1 {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
data, err := json.Marshal(summary)
|
|
if err != nil {
|
|
t.Fatalf("Marshal() error = %v", err)
|
|
}
|
|
for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
|
|
if strings.Contains(string(data), forbidden) {
|
|
t.Fatalf("summary includes %q: %s", forbidden, data)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestComparisonSummaryUsesLockedOrderAndSafeFields(t *testing.T) {
|
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
profileFailure := comparison.NewSafeError("generation", "execute prompt failed")
|
|
result := &app.ComparisonResult{
|
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, ReportName: "Daily Report",
|
|
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
|
StartedAt: started, FinishedAt: started.Add(time.Minute), Timezone: "America/Chicago",
|
|
ValidPeriod: timeutil.Period{Start: started, End: started.Add(24 * time.Hour)}, OutputDirectory: "/reports/comparison-daily",
|
|
ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
|
Total: 2, Succeeded: 1, Failed: 1,
|
|
Results: []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", BackendID: "local", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
|
{Position: 2, ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep", Status: comparison.StatusFailed, Error: &profileFailure},
|
|
},
|
|
}
|
|
summary := newComparisonSummary(result, errors.New("comparison completed with 1 failed profiles"))
|
|
if summary.Command != commandCompare || summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 1 failed profiles" || len(summary.Results) != 2 || summary.Results[0].ReportPath == "" || summary.Results[1].Error == nil {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
data, err := json.Marshal(summary)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previous := -1
|
|
for _, field := range []string{"command", "comparisonId", "reportId", "reportName", "promptId", "promptVersion", "promptHash", "status", "startedAt", "finishedAt", "timezone", "validPeriod", "outputDirectory", "manifestPath", "dataPackagePath", "total", "succeeded", "failed", "results", "error"} {
|
|
position := strings.Index(string(data), `"`+field+`":`)
|
|
if field == "error" {
|
|
position = strings.LastIndex(string(data), `"`+field+`":`)
|
|
}
|
|
if position <= previous {
|
|
t.Fatalf("field order for %q in %s", field, data)
|
|
}
|
|
previous = position
|
|
}
|
|
for _, unsafe := range []string{"provider response", "rawOutput", "renderedPrompt", "endpoint"} {
|
|
if strings.Contains(string(data), unsafe) {
|
|
t.Fatalf("summary includes unsafe content %q: %s", unsafe, data)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestComparisonSummaryOmitsUnpublishedArtifactsAndBoundsErrors(t *testing.T) {
|
|
result := &app.ComparisonResult{ComparisonID: "comparison_run-123", ReportID: report.Daily, Results: []app.ComparisonProfileResult{{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed}}}
|
|
summary := newComparisonSummary(result, context.Canceled)
|
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison canceled" || summary.Results == nil {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
data, err := json.Marshal(summary)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(data, &fields); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, omitted := range []string{"manifestPath", "dataPackagePath"} {
|
|
if _, exists := fields[omitted]; exists {
|
|
t.Fatalf("summary includes unpublished %s: %s", omitted, data)
|
|
}
|
|
}
|
|
publicationResult := &app.ComparisonResult{
|
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, OutputDirectory: "/reports/comparison-daily", Total: 2, Succeeded: 2,
|
|
Results: []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded},
|
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded},
|
|
},
|
|
}
|
|
unsafe := errors.New("comparison completed with 1 failed profiles; provider response contains sensitive material")
|
|
publicationSummary := newComparisonSummary(publicationResult, unsafe)
|
|
if publicationSummary.Status != summaryStatusFailed || publicationSummary.Error == nil || publicationSummary.Error.Message != "comparison did not complete" || strings.Contains(publicationSummary.Error.Message, "sensitive") || publicationSummary.ManifestPath != "" || publicationSummary.DataPackagePath != "" {
|
|
safe := publicationSummary.Error
|
|
t.Fatalf("safe error = %#v", safe)
|
|
}
|
|
}
|
|
|
|
func TestComparisonSummaryClassifiesCompleteAndAllFailedResults(t *testing.T) {
|
|
started := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
failure := comparison.NewSafeError("generation", "execute prompt failed")
|
|
complete := &app.ComparisonResult{
|
|
ComparisonID: "comparison_run-123", ReportID: report.Daily, PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: strings.Repeat("a", 64),
|
|
StartedAt: started, FinishedAt: started, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: started, End: started.Add(time.Hour)},
|
|
OutputDirectory: "/reports/comparison-daily", ManifestPath: "/reports/comparison-daily/comparison.json", DataPackagePath: "/reports/comparison-daily/data-package.yml",
|
|
Total: 2, Succeeded: 2,
|
|
Results: []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/01-weather-light.md"},
|
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusSucceeded, ValidationStatus: promptexec.ValidationPassed, ReportPath: "/reports/comparison-daily/02-weather-deep.md"},
|
|
},
|
|
}
|
|
if summary := newComparisonSummary(complete, nil); summary.Status != summaryStatusSucceeded || summary.Error != nil {
|
|
t.Fatalf("complete summary = %#v", summary)
|
|
}
|
|
allFailed := *complete
|
|
allFailed.Succeeded, allFailed.Failed = 0, 2
|
|
allFailed.Results = []app.ComparisonProfileResult{
|
|
{Position: 1, ProfileID: "weather-light", ModelName: "light", Status: comparison.StatusFailed, Error: &failure},
|
|
{Position: 2, ProfileID: "weather-deep", ModelName: "deep", Status: comparison.StatusFailed, Error: &failure},
|
|
}
|
|
summary := newComparisonSummary(&allFailed, errors.New("comparison completed with 2 failed profiles"))
|
|
if summary.Status != summaryStatusFailed || summary.Error == nil || summary.Error.Message != "comparison completed with 2 failed profiles" || summary.Results[0].ReportPath != "" || summary.Results[1].Error == nil {
|
|
t.Fatalf("all-failed summary = %#v", summary)
|
|
}
|
|
}
|
|
|
|
func TestSafeComparisonSummaryErrorClassifiesWrappedFailures(t *testing.T) {
|
|
unsafeDetail := "unsafe filesystem and provider detail"
|
|
unsafeCause := errors.New(unsafeDetail)
|
|
for _, test := range []struct {
|
|
name string
|
|
err error
|
|
category string
|
|
message string
|
|
}{
|
|
{
|
|
name: "aggregate profile failure",
|
|
err: fmt.Errorf("outer wrapper: %w", errors.New("comparison completed with 2 failed profiles")),
|
|
category: "application",
|
|
message: "comparison completed with 2 failed profiles",
|
|
},
|
|
{
|
|
name: "canceled",
|
|
err: fmt.Errorf("outer wrapper: %w", context.Canceled),
|
|
category: "canceled",
|
|
message: "comparison canceled",
|
|
},
|
|
{
|
|
name: "deadline exceeded",
|
|
err: fmt.Errorf("outer wrapper: %w", context.DeadlineExceeded),
|
|
category: "deadline_exceeded",
|
|
message: "comparison deadline exceeded",
|
|
},
|
|
{
|
|
name: "prompt operation",
|
|
err: fmt.Errorf("outer wrapper: %w", promptexec.NewError(promptexec.Generation, "unsafe prompt detail", fmt.Errorf("%w: %s", context.Canceled, unsafeDetail))),
|
|
category: string(promptexec.Generation),
|
|
message: "comparison prompt operation failed",
|
|
},
|
|
{
|
|
name: "destination preflight",
|
|
err: fmt.Errorf("outer wrapper: %w", &comparison.DestinationError{
|
|
Kind: comparison.DestinationNotEmpty, Target: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
|
}),
|
|
category: "destination_not_empty",
|
|
message: "comparison destination preflight failed",
|
|
},
|
|
{
|
|
name: "complete cleanup recovery",
|
|
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
|
RecoveryState: comparison.BackupRecoveryComplete, RecoveryPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
|
}),
|
|
category: "publication_cleanup",
|
|
message: "comparison published but a complete prior bundle remains",
|
|
},
|
|
{
|
|
name: "partial cleanup remnants",
|
|
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
|
RecoveryState: comparison.BackupRecoveryPartial, RecoveryPath: "/tmp/" + unsafeDetail, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
|
}),
|
|
category: "publication_cleanup",
|
|
message: "comparison published but partial cleanup remnants remain",
|
|
},
|
|
{
|
|
name: "absent cleanup recovery",
|
|
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
|
RecoveryState: comparison.BackupRecoveryAbsent, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
|
}),
|
|
category: "publication_cleanup",
|
|
message: "comparison published but no prior bundle remains",
|
|
},
|
|
{
|
|
name: "unknown cleanup recovery",
|
|
err: fmt.Errorf("outer wrapper: %w", &comparison.PublicationCleanupError{
|
|
RecoveryState: comparison.BackupRecoveryUnknown, Err: fmt.Errorf("%w: %s", context.Canceled, unsafeDetail),
|
|
}),
|
|
category: "publication_cleanup",
|
|
message: "comparison published but cleanup recovery state is unknown",
|
|
},
|
|
{
|
|
name: "unknown",
|
|
err: fmt.Errorf("outer wrapper: %w", unsafeCause),
|
|
category: "application",
|
|
message: "comparison did not complete",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
result := &app.ComparisonResult{ComparisonID: "comparison_test", ReportID: report.Daily}
|
|
summary := newComparisonSummary(result, test.err)
|
|
if summary.Error == nil || summary.Error.Category != test.category || summary.Error.Message != test.message {
|
|
t.Fatalf("summary error = %#v, want %q/%q", summary.Error, test.category, test.message)
|
|
}
|
|
data, err := json.Marshal(summary)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(data), unsafeDetail) {
|
|
t.Fatalf("summary includes unsafe detail: %s", data)
|
|
}
|
|
})
|
|
}
|
|
}
|