package comparison import ( "encoding/json" "strings" "testing" "time" "unicode/utf8" ) func TestValidateProfileIDs(t *testing.T) { t.Parallel() tests := []struct { name string profileIDs []string wantErr bool }{ {name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}}, {name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}}, {name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true}, {name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true}, {name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() err := ValidateProfileIDs(test.profileIDs) if (err != nil) != test.wantErr { t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr) } }) } } func TestProfileSlug(t *testing.T) { t.Parallel() long := strings.Repeat("a", 62) + "-_more" tests := []struct { profileID string want string }{ {profileID: "weather-light", want: "weather-light"}, {profileID: "model.v1", want: "model-v1"}, {profileID: "nested/path\\name", want: "nested-path-name"}, {profileID: " weather\tlight\n", want: "weather-light"}, {profileID: "\x00\x01", want: "profile"}, {profileID: "météo-東京", want: "m-t-o"}, {profileID: "___", want: "profile"}, {profileID: long, want: strings.Repeat("a", 62)}, } for _, test := range tests { t.Run(test.profileID, func(t *testing.T) { t.Parallel() if got := ProfileSlug(test.profileID); got != test.want { t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want) } }) } } func TestReportNaming(t *testing.T) { t.Parallel() if got := OrdinalWidth(99); got != 2 { t.Fatalf("OrdinalWidth(99) = %d, want 2", got) } if got := OrdinalWidth(100); got != 3 { t.Fatalf("OrdinalWidth(100) = %d, want 3", got) } if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" { t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md") } if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" { t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md") } first, err := ReportFilename(1, 2, "model.v1") if err != nil { t.Fatalf("first ReportFilename() error = %v", err) } second, err := ReportFilename(2, 2, "model/v1") if err != nil { t.Fatalf("second ReportFilename() error = %v", err) } if first == second { t.Fatalf("normalized profile collisions produced the same filename %q", first) } if _, err := ReportFilename(0, 2, "weather-light"); err == nil { t.Fatal("ReportFilename accepted position zero") } if _, err := ReportFilename(1, 0, "weather-light"); err == nil { t.Fatal("ReportFilename accepted zero profile count") } if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" { t.Fatalf("BuildComparisonID() = %q, %v", got, err) } if _, err := BuildComparisonID(" \t"); err == nil { t.Fatal("BuildComparisonID accepted blank run ID") } if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" { t.Fatalf("DefaultDirectoryName() = %q, %v", got, err) } for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} { if _, err := DefaultDirectoryName(name); err == nil { t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name) } } } func TestManifestEncodingAndRoundTrip(t *testing.T) { t.Parallel() manifest := validManifest() encoded, err := EncodeManifest(manifest) if err != nil { t.Fatalf("EncodeManifest() error = %v", err) } want := "{\n" + " \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" + " \"comparisonId\": \"comparison_daily-2026-08-24\",\n" + " \"startedAt\": \"2026-08-24T12:00:00Z\",\n" + " \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" + " \"reportId\": \"daily\",\n" + " \"validPeriod\": {\n" + " \"start\": \"2026-08-24T00:00:00-04:00\",\n" + " \"end\": \"2026-08-25T00:00:00-04:00\"\n" + " },\n" + " \"timezone\": \"America/New_York\",\n" + " \"promptId\": \"daily-report\",\n" + " \"promptVersion\": \"2026-08-01\",\n" + " \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" + " \"dataPackage\": {\n" + " \"path\": \"data-package.yml\",\n" + " \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" + " },\n" + " \"total\": 2,\n" + " \"succeeded\": 1,\n" + " \"failed\": 1,\n" + " \"results\": [\n" + " {\n" + " \"position\": 1,\n" + " \"profileId\": \"weather-light\",\n" + " \"backendId\": \"openai\",\n" + " \"modelName\": \"gpt-5-mini\",\n" + " \"status\": \"succeeded\",\n" + " \"validationStatus\": \"passed\",\n" + " \"reportPath\": \"01-weather-light.md\"\n" + " },\n" + " {\n" + " \"position\": 2,\n" + " \"profileId\": \"weather-deep\",\n" + " \"modelName\": \"gpt-5\",\n" + " \"status\": \"failed\",\n" + " \"error\": {\n" + " \"category\": \"application\",\n" + " \"message\": \"generated text was rejected\"\n" + " }\n" + " }\n" + " ]\n" + "}\n" if string(encoded) != want { t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want) } var decoded Manifest if err := json.Unmarshal(encoded, &decoded); err != nil { t.Fatalf("json.Unmarshal() error = %v", err) } if err := decoded.Validate(); err != nil { t.Fatalf("decoded manifest validation error = %v", err) } } func TestManifestValidateRejectsInvariants(t *testing.T) { t.Parallel() tests := []struct { name string mutate func(*Manifest) }{ {name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }}, {name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }}, {name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }}, {name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }}, {name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }}, {name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }}, {name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }}, {name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }}, {name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }}, {name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }}, {name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }}, {name: "successful result with error", mutate: func(manifest *Manifest) { manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"} }}, {name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }}, {name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }}, {name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }}, {name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }}, {name: "duplicate report path", mutate: func(manifest *Manifest) { manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath} manifest.Succeeded = 2 manifest.Failed = 0 }}, {name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() manifest := validManifest() test.mutate(&manifest) if err := manifest.Validate(); err == nil { t.Fatal("Manifest.Validate() succeeded for invalid manifest") } }) } } func TestLogicalBundleValidate(t *testing.T) { t.Parallel() dataPackage := []byte("report: daily\n") manifest := validManifest() manifest.DataPackage.SHA256 = SHA256(dataPackage) bundle := LogicalBundle{ Manifest: manifest, DataPackage: dataPackage, Reports: []BundleReport{{ Position: 1, Path: "01-weather-light.md", Markdown: []byte("# Daily\n"), }}, } if err := bundle.Validate(); err != nil { t.Fatalf("LogicalBundle.Validate() error = %v", err) } bundle.Reports[0].Path = "other.md" if err := bundle.Validate(); err == nil { t.Fatal("LogicalBundle.Validate() accepted mismatched report path") } bundle.Reports[0].Path = "01-weather-light.md" bundle.Reports[0].Position = 2 if err := bundle.Validate(); err == nil { t.Fatal("LogicalBundle.Validate() accepted unordered report position") } } func TestSHA256AndTruncateErrorMessage(t *testing.T) { t.Parallel() if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want { t.Fatalf("SHA256() = %q, want %q", got, want) } message := strings.Repeat("€", 400) got := TruncateErrorMessage(message) if len(got) > 1024 || !utf8.ValidString(got) { t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got)) } if want := strings.Repeat("€", 341); got != want { t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want) } invalid := string([]byte{'x', 0xff, 'y'}) if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) { t.Fatal("TruncateErrorMessage() retained invalid UTF-8") } } func validManifest() Manifest { newYork := time.FixedZone("-0400", -4*60*60) return Manifest{ SchemaVersion: SchemaVersion, ComparisonID: "comparison_daily-2026-08-24", StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC), FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC), ReportID: "daily", ValidPeriod: ValidPeriod{ Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork), End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork), }, Timezone: "America/New_York", PromptID: "daily-report", PromptVersion: "2026-08-01", PromptHash: strings.Repeat("a", 64), DataPackage: DataPackageReference{ Path: DataPackageFilename, SHA256: strings.Repeat("a", 64), }, Total: 2, Succeeded: 1, Failed: 1, Results: []Result{ { Position: 1, ProfileID: "weather-light", BackendID: "openai", ModelName: "gpt-5-mini", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: "01-weather-light.md", }, { Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusFailed, Error: &SafeError{Category: "application", Message: "generated text was rejected"}, }, }, } }