324 lines
12 KiB
Go
324 lines
12 KiB
Go
package comparison
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func TestValidateProfileIDs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
profileIDs []string
|
|
wantErr bool
|
|
}{
|
|
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
|
|
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
|
|
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
|
|
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
|
|
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
err := ValidateProfileIDs(test.profileIDs)
|
|
if (err != nil) != test.wantErr {
|
|
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProfileSlug(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
long := strings.Repeat("a", 62) + "-_more"
|
|
tests := []struct {
|
|
profileID string
|
|
want string
|
|
}{
|
|
{profileID: "weather-light", want: "weather-light"},
|
|
{profileID: "model.v1", want: "model-v1"},
|
|
{profileID: "nested/path\\name", want: "nested-path-name"},
|
|
{profileID: " weather\tlight\n", want: "weather-light"},
|
|
{profileID: "\x00\x01", want: "profile"},
|
|
{profileID: "météo-東京", want: "m-t-o"},
|
|
{profileID: "___", want: "profile"},
|
|
{profileID: long, want: strings.Repeat("a", 62)},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.profileID, func(t *testing.T) {
|
|
t.Parallel()
|
|
if got := ProfileSlug(test.profileID); got != test.want {
|
|
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReportNaming(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := OrdinalWidth(99); got != 2 {
|
|
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
|
|
}
|
|
if got := OrdinalWidth(100); got != 3 {
|
|
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
|
|
}
|
|
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
|
|
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
|
|
}
|
|
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
|
|
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
|
|
}
|
|
first, err := ReportFilename(1, 2, "model.v1")
|
|
if err != nil {
|
|
t.Fatalf("first ReportFilename() error = %v", err)
|
|
}
|
|
second, err := ReportFilename(2, 2, "model/v1")
|
|
if err != nil {
|
|
t.Fatalf("second ReportFilename() error = %v", err)
|
|
}
|
|
if first == second {
|
|
t.Fatalf("normalized profile collisions produced the same filename %q", first)
|
|
}
|
|
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
|
|
t.Fatal("ReportFilename accepted position zero")
|
|
}
|
|
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
|
|
t.Fatal("ReportFilename accepted zero profile count")
|
|
}
|
|
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
|
|
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
|
|
}
|
|
if _, err := BuildComparisonID(" \t"); err == nil {
|
|
t.Fatal("BuildComparisonID accepted blank run ID")
|
|
}
|
|
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
|
|
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
|
|
}
|
|
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
|
|
if _, err := DefaultDirectoryName(name); err == nil {
|
|
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManifestEncodingAndRoundTrip(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
manifest := validManifest()
|
|
encoded, err := EncodeManifest(manifest)
|
|
if err != nil {
|
|
t.Fatalf("EncodeManifest() error = %v", err)
|
|
}
|
|
want := "{\n" +
|
|
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
|
|
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
|
|
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
|
|
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
|
|
" \"reportId\": \"daily\",\n" +
|
|
" \"validPeriod\": {\n" +
|
|
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
|
|
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
|
|
" },\n" +
|
|
" \"timezone\": \"America/New_York\",\n" +
|
|
" \"promptId\": \"daily-report\",\n" +
|
|
" \"promptVersion\": \"2026-08-01\",\n" +
|
|
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
|
|
" \"dataPackage\": {\n" +
|
|
" \"path\": \"data-package.yml\",\n" +
|
|
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
|
|
" },\n" +
|
|
" \"total\": 2,\n" +
|
|
" \"succeeded\": 1,\n" +
|
|
" \"failed\": 1,\n" +
|
|
" \"results\": [\n" +
|
|
" {\n" +
|
|
" \"position\": 1,\n" +
|
|
" \"profileId\": \"weather-light\",\n" +
|
|
" \"backendId\": \"openai\",\n" +
|
|
" \"modelName\": \"gpt-5-mini\",\n" +
|
|
" \"status\": \"succeeded\",\n" +
|
|
" \"validationStatus\": \"passed\",\n" +
|
|
" \"reportPath\": \"01-weather-light.md\"\n" +
|
|
" },\n" +
|
|
" {\n" +
|
|
" \"position\": 2,\n" +
|
|
" \"profileId\": \"weather-deep\",\n" +
|
|
" \"modelName\": \"gpt-5\",\n" +
|
|
" \"status\": \"failed\",\n" +
|
|
" \"error\": {\n" +
|
|
" \"category\": \"application\",\n" +
|
|
" \"message\": \"generated text was rejected\"\n" +
|
|
" }\n" +
|
|
" }\n" +
|
|
" ]\n" +
|
|
"}\n"
|
|
if string(encoded) != want {
|
|
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
|
|
}
|
|
|
|
decoded, err := decodeManifest(encoded)
|
|
if err != nil {
|
|
t.Fatalf("decodeManifest() error = %v", err)
|
|
}
|
|
if err := decoded.Validate(); err != nil {
|
|
t.Fatalf("decoded manifest validation error = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestManifestValidateRejectsInvariants(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*Manifest)
|
|
}{
|
|
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
|
|
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
|
|
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
|
|
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
|
|
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
|
|
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
|
|
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
|
|
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
|
|
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
|
|
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
|
|
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
|
|
{name: "arbitrary report name", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "arbitrary.md" }},
|
|
{name: "wrong report position", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "02-weather-light.md" }},
|
|
{name: "wrong report ordinal width", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "1-weather-light.md" }},
|
|
{name: "wrong report profile slug", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "01-weather.md" }},
|
|
{name: "successful result with error", mutate: func(manifest *Manifest) {
|
|
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
|
|
}},
|
|
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
|
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
|
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
|
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
|
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
|
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
|
manifest.Succeeded = 2
|
|
manifest.Failed = 0
|
|
}},
|
|
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
manifest := validManifest()
|
|
test.mutate(&manifest)
|
|
if err := manifest.Validate(); err == nil {
|
|
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLogicalBundleValidate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dataPackage := []byte("report: daily\n")
|
|
manifest := validManifest()
|
|
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
|
bundle := LogicalBundle{
|
|
Manifest: manifest,
|
|
DataPackage: dataPackage,
|
|
Reports: []BundleReport{{
|
|
Position: 1,
|
|
Path: "01-weather-light.md",
|
|
Markdown: []byte("# Daily\n"),
|
|
}},
|
|
}
|
|
if err := bundle.Validate(); err != nil {
|
|
t.Fatalf("LogicalBundle.Validate() error = %v", err)
|
|
}
|
|
|
|
bundle.Reports[0].Path = "other.md"
|
|
if err := bundle.Validate(); err == nil {
|
|
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
|
|
}
|
|
bundle.Reports[0].Path = "01-weather-light.md"
|
|
bundle.Reports[0].Position = 2
|
|
if err := bundle.Validate(); err == nil {
|
|
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
|
|
}
|
|
bundle.Reports[0].Position = 1
|
|
bundle.Reports[0].Path = "arbitrary.md"
|
|
bundle.Manifest.Results[0].ReportPath = "arbitrary.md"
|
|
if err := bundle.Validate(); err == nil {
|
|
t.Fatal("LogicalBundle.Validate() accepted a noncanonical report path")
|
|
}
|
|
}
|
|
|
|
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
|
|
t.Fatalf("SHA256() = %q, want %q", got, want)
|
|
}
|
|
message := strings.Repeat("€", 400)
|
|
got := TruncateErrorMessage(message)
|
|
if len(got) > 1024 || !utf8.ValidString(got) {
|
|
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
|
|
}
|
|
if want := strings.Repeat("€", 341); got != want {
|
|
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
|
|
}
|
|
invalid := string([]byte{'x', 0xff, 'y'})
|
|
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
|
|
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
|
|
}
|
|
}
|
|
|
|
func validManifest() Manifest {
|
|
newYork := time.FixedZone("-0400", -4*60*60)
|
|
return Manifest{
|
|
SchemaVersion: SchemaVersion,
|
|
ComparisonID: "comparison_daily-2026-08-24",
|
|
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
|
|
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
|
|
ReportID: "daily",
|
|
ValidPeriod: ValidPeriod{
|
|
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
|
|
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
|
|
},
|
|
Timezone: "America/New_York",
|
|
PromptID: "daily-report",
|
|
PromptVersion: "2026-08-01",
|
|
PromptHash: strings.Repeat("a", 64),
|
|
DataPackage: DataPackageReference{
|
|
Path: DataPackageFilename,
|
|
SHA256: strings.Repeat("a", 64),
|
|
},
|
|
Total: 2,
|
|
Succeeded: 1,
|
|
Failed: 1,
|
|
Results: []Result{
|
|
{
|
|
Position: 1,
|
|
ProfileID: "weather-light",
|
|
BackendID: "openai",
|
|
ModelName: "gpt-5-mini",
|
|
Status: StatusSucceeded,
|
|
ValidationStatus: "passed",
|
|
ReportPath: "01-weather-light.md",
|
|
},
|
|
{
|
|
Position: 2,
|
|
ProfileID: "weather-deep",
|
|
ModelName: "gpt-5",
|
|
Status: StatusFailed,
|
|
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
|
|
},
|
|
},
|
|
}
|
|
}
|