Define comparison artifact contracts
This commit is contained in:
314
internal/comparison/comparison_test.go
Normal file
314
internal/comparison/comparison_test.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestValidateProfileIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
profileIDs []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
|
||||
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
|
||||
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
|
||||
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
|
||||
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := ValidateProfileIDs(test.profileIDs)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileSlug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
long := strings.Repeat("a", 62) + "-_more"
|
||||
tests := []struct {
|
||||
profileID string
|
||||
want string
|
||||
}{
|
||||
{profileID: "weather-light", want: "weather-light"},
|
||||
{profileID: "model.v1", want: "model-v1"},
|
||||
{profileID: "nested/path\\name", want: "nested-path-name"},
|
||||
{profileID: " weather\tlight\n", want: "weather-light"},
|
||||
{profileID: "\x00\x01", want: "profile"},
|
||||
{profileID: "météo-東京", want: "m-t-o"},
|
||||
{profileID: "___", want: "profile"},
|
||||
{profileID: long, want: strings.Repeat("a", 62)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.profileID, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := ProfileSlug(test.profileID); got != test.want {
|
||||
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportNaming(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := OrdinalWidth(99); got != 2 {
|
||||
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
|
||||
}
|
||||
if got := OrdinalWidth(100); got != 3 {
|
||||
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
|
||||
}
|
||||
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
|
||||
}
|
||||
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
|
||||
}
|
||||
first, err := ReportFilename(1, 2, "model.v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first ReportFilename() error = %v", err)
|
||||
}
|
||||
second, err := ReportFilename(2, 2, "model/v1")
|
||||
if err != nil {
|
||||
t.Fatalf("second ReportFilename() error = %v", err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("normalized profile collisions produced the same filename %q", first)
|
||||
}
|
||||
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted position zero")
|
||||
}
|
||||
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted zero profile count")
|
||||
}
|
||||
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
|
||||
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
|
||||
}
|
||||
if _, err := BuildComparisonID(" \t"); err == nil {
|
||||
t.Fatal("BuildComparisonID accepted blank run ID")
|
||||
}
|
||||
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
|
||||
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
|
||||
}
|
||||
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
|
||||
if _, err := DefaultDirectoryName(name); err == nil {
|
||||
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestEncodingAndRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := validManifest()
|
||||
encoded, err := EncodeManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeManifest() error = %v", err)
|
||||
}
|
||||
want := "{\n" +
|
||||
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
|
||||
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
|
||||
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
|
||||
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
|
||||
" \"reportId\": \"daily\",\n" +
|
||||
" \"validPeriod\": {\n" +
|
||||
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
|
||||
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
|
||||
" },\n" +
|
||||
" \"timezone\": \"America/New_York\",\n" +
|
||||
" \"promptId\": \"daily-report\",\n" +
|
||||
" \"promptVersion\": \"2026-08-01\",\n" +
|
||||
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
|
||||
" \"dataPackage\": {\n" +
|
||||
" \"path\": \"data-package.yml\",\n" +
|
||||
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
|
||||
" },\n" +
|
||||
" \"total\": 2,\n" +
|
||||
" \"succeeded\": 1,\n" +
|
||||
" \"failed\": 1,\n" +
|
||||
" \"results\": [\n" +
|
||||
" {\n" +
|
||||
" \"position\": 1,\n" +
|
||||
" \"profileId\": \"weather-light\",\n" +
|
||||
" \"backendId\": \"openai\",\n" +
|
||||
" \"modelName\": \"gpt-5-mini\",\n" +
|
||||
" \"status\": \"succeeded\",\n" +
|
||||
" \"validationStatus\": \"passed\",\n" +
|
||||
" \"reportPath\": \"01-weather-light.md\"\n" +
|
||||
" },\n" +
|
||||
" {\n" +
|
||||
" \"position\": 2,\n" +
|
||||
" \"profileId\": \"weather-deep\",\n" +
|
||||
" \"modelName\": \"gpt-5\",\n" +
|
||||
" \"status\": \"failed\",\n" +
|
||||
" \"error\": {\n" +
|
||||
" \"category\": \"application\",\n" +
|
||||
" \"message\": \"generated text was rejected\"\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" ]\n" +
|
||||
"}\n"
|
||||
if string(encoded) != want {
|
||||
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
|
||||
}
|
||||
|
||||
var decoded Manifest
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if err := decoded.Validate(); err != nil {
|
||||
t.Fatalf("decoded manifest validation error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Manifest)
|
||||
}{
|
||||
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
|
||||
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
|
||||
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
|
||||
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
|
||||
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
|
||||
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
|
||||
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
|
||||
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
|
||||
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
|
||||
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
|
||||
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
|
||||
{name: "successful result with error", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
|
||||
}},
|
||||
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
||||
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
||||
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
||||
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
||||
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
||||
manifest.Succeeded = 2
|
||||
manifest.Failed = 0
|
||||
}},
|
||||
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
manifest := validManifest()
|
||||
test.mutate(&manifest)
|
||||
if err := manifest.Validate(); err == nil {
|
||||
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalBundleValidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataPackage := []byte("report: daily\n")
|
||||
manifest := validManifest()
|
||||
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||
bundle := LogicalBundle{
|
||||
Manifest: manifest,
|
||||
DataPackage: dataPackage,
|
||||
Reports: []BundleReport{{
|
||||
Position: 1,
|
||||
Path: "01-weather-light.md",
|
||||
Markdown: []byte("# Daily\n"),
|
||||
}},
|
||||
}
|
||||
if err := bundle.Validate(); err != nil {
|
||||
t.Fatalf("LogicalBundle.Validate() error = %v", err)
|
||||
}
|
||||
|
||||
bundle.Reports[0].Path = "other.md"
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
|
||||
}
|
||||
bundle.Reports[0].Path = "01-weather-light.md"
|
||||
bundle.Reports[0].Position = 2
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
|
||||
t.Fatalf("SHA256() = %q, want %q", got, want)
|
||||
}
|
||||
message := strings.Repeat("€", 400)
|
||||
got := TruncateErrorMessage(message)
|
||||
if len(got) > 1024 || !utf8.ValidString(got) {
|
||||
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
|
||||
}
|
||||
if want := strings.Repeat("€", 341); got != want {
|
||||
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
|
||||
}
|
||||
invalid := string([]byte{'x', 0xff, 'y'})
|
||||
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
|
||||
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
func validManifest() Manifest {
|
||||
newYork := time.FixedZone("-0400", -4*60*60)
|
||||
return Manifest{
|
||||
SchemaVersion: SchemaVersion,
|
||||
ComparisonID: "comparison_daily-2026-08-24",
|
||||
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
|
||||
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
|
||||
ReportID: "daily",
|
||||
ValidPeriod: ValidPeriod{
|
||||
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
|
||||
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
|
||||
},
|
||||
Timezone: "America/New_York",
|
||||
PromptID: "daily-report",
|
||||
PromptVersion: "2026-08-01",
|
||||
PromptHash: strings.Repeat("a", 64),
|
||||
DataPackage: DataPackageReference{
|
||||
Path: DataPackageFilename,
|
||||
SHA256: strings.Repeat("a", 64),
|
||||
},
|
||||
Total: 2,
|
||||
Succeeded: 1,
|
||||
Failed: 1,
|
||||
Results: []Result{
|
||||
{
|
||||
Position: 1,
|
||||
ProfileID: "weather-light",
|
||||
BackendID: "openai",
|
||||
ModelName: "gpt-5-mini",
|
||||
Status: StatusSucceeded,
|
||||
ValidationStatus: "passed",
|
||||
ReportPath: "01-weather-light.md",
|
||||
},
|
||||
{
|
||||
Position: 2,
|
||||
ProfileID: "weather-deep",
|
||||
ModelName: "gpt-5",
|
||||
Status: StatusFailed,
|
||||
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user