Define comparison artifact contracts
This commit is contained in:
386
internal/comparison/comparison.go
Normal file
386
internal/comparison/comparison.go
Normal file
@@ -0,0 +1,386 @@
|
||||
// Package comparison defines the durable logical contract for profile
|
||||
// comparison bundles. It deliberately has no filesystem or application
|
||||
// orchestration dependencies.
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
// SchemaVersion identifies the supported comparison manifest schema.
|
||||
SchemaVersion = "weatherreporter.comparison.v1"
|
||||
|
||||
// ManifestFilename is the canonical name of a comparison manifest.
|
||||
ManifestFilename = "comparison.json"
|
||||
// DataPackageFilename is the canonical name of the shared prompt input.
|
||||
DataPackageFilename = "data-package.yml"
|
||||
|
||||
// StatusSucceeded identifies a profile that produced a validated report.
|
||||
StatusSucceeded = "succeeded"
|
||||
// StatusFailed identifies a profile that did not produce a report.
|
||||
StatusFailed = "failed"
|
||||
|
||||
maxProfileSlugBytes = 64
|
||||
maxErrorMessageBytes = 1024
|
||||
)
|
||||
|
||||
// Manifest is the versioned, authoritative index of a comparison bundle.
|
||||
// Field declaration order is the JSON field order.
|
||||
type Manifest struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
ComparisonID string `json:"comparisonId"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt"`
|
||||
ReportID string `json:"reportId"`
|
||||
ValidPeriod ValidPeriod `json:"validPeriod"`
|
||||
Timezone string `json:"timezone"`
|
||||
PromptID string `json:"promptId"`
|
||||
PromptVersion string `json:"promptVersion"`
|
||||
PromptHash string `json:"promptHash"`
|
||||
DataPackage DataPackageReference `json:"dataPackage"`
|
||||
Total int `json:"total"`
|
||||
Succeeded int `json:"succeeded"`
|
||||
Failed int `json:"failed"`
|
||||
Results []Result `json:"results"`
|
||||
}
|
||||
|
||||
// ValidPeriod records the report's resolved half-open period.
|
||||
type ValidPeriod struct {
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
}
|
||||
|
||||
// DataPackageReference identifies and verifies the shared prompt input.
|
||||
type DataPackageReference struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
// Result records one explicitly selected profile in selection order.
|
||||
type Result struct {
|
||||
Position int `json:"position"`
|
||||
ProfileID string `json:"profileId"`
|
||||
BackendID string `json:"backendId,omitempty"`
|
||||
ModelName string `json:"modelName"`
|
||||
Status string `json:"status"`
|
||||
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||
ReportPath string `json:"reportPath,omitempty"`
|
||||
Error *SafeError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// SafeError contains bounded, operator-safe failure information only.
|
||||
type SafeError struct {
|
||||
Category string `json:"category"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// LogicalBundle contains every byte required to publish a comparison without
|
||||
// coupling the comparison contract to a filesystem implementation.
|
||||
type LogicalBundle struct {
|
||||
Manifest Manifest
|
||||
DataPackage []byte
|
||||
Reports []BundleReport
|
||||
}
|
||||
|
||||
// BundleReport is one rendered Markdown document in a logical bundle.
|
||||
type BundleReport struct {
|
||||
Position int
|
||||
Path string
|
||||
Markdown []byte
|
||||
}
|
||||
|
||||
// ValidateProfileIDs requires at least two distinct, nonblank profile IDs. It
|
||||
// intentionally preserves accepted IDs unchanged and treats case distinctly.
|
||||
func ValidateProfileIDs(profileIDs []string) error {
|
||||
if len(profileIDs) < 2 {
|
||||
return fmt.Errorf("comparison requires at least two profile IDs")
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(profileIDs))
|
||||
for _, profileID := range profileIDs {
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return fmt.Errorf("profile ID must not be blank")
|
||||
}
|
||||
if _, duplicate := seen[profileID]; duplicate {
|
||||
return fmt.Errorf("duplicate profile ID %q", profileID)
|
||||
}
|
||||
seen[profileID] = struct{}{}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProfileSlug returns a deterministic, filesystem-safe representation of a
|
||||
// logical profile ID. The logical ID remains authoritative in the manifest.
|
||||
func ProfileSlug(profileID string) string {
|
||||
var builder strings.Builder
|
||||
lastReplacement := false
|
||||
for _, r := range profileID {
|
||||
if isSlugRune(r) {
|
||||
builder.WriteRune(r)
|
||||
lastReplacement = false
|
||||
continue
|
||||
}
|
||||
if !lastReplacement {
|
||||
builder.WriteByte('-')
|
||||
lastReplacement = true
|
||||
}
|
||||
}
|
||||
|
||||
slug := strings.Trim(builder.String(), "-_")
|
||||
if len(slug) > maxProfileSlugBytes {
|
||||
slug = strings.Trim(slug[:maxProfileSlugBytes], "-_")
|
||||
}
|
||||
if slug == "" {
|
||||
return "profile"
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
func isSlugRune(r rune) bool {
|
||||
return r >= 'a' && r <= 'z' ||
|
||||
r >= 'A' && r <= 'Z' ||
|
||||
r >= '0' && r <= '9' ||
|
||||
r == '-' || r == '_'
|
||||
}
|
||||
|
||||
// OrdinalWidth returns the zero-padding width for a comparison of count
|
||||
// profiles.
|
||||
func OrdinalWidth(count int) int {
|
||||
width := 2
|
||||
for value := count; value >= 100; value /= 10 {
|
||||
width++
|
||||
}
|
||||
return width
|
||||
}
|
||||
|
||||
// ReportFilename derives a report's deterministic bundle filename.
|
||||
func ReportFilename(position, profileCount int, profileID string) (string, error) {
|
||||
if profileCount < 1 {
|
||||
return "", fmt.Errorf("profile count must be positive")
|
||||
}
|
||||
if position < 1 || position > profileCount {
|
||||
return "", fmt.Errorf("profile position %d is outside 1..%d", position, profileCount)
|
||||
}
|
||||
return fmt.Sprintf("%0*d-%s.md", OrdinalWidth(profileCount), position, ProfileSlug(profileID)), nil
|
||||
}
|
||||
|
||||
// BuildComparisonID derives the stable comparison identity for a resolved
|
||||
// report run.
|
||||
func BuildComparisonID(reportRunID string) (string, error) {
|
||||
if strings.TrimSpace(reportRunID) == "" {
|
||||
return "", fmt.Errorf("report run ID must not be blank")
|
||||
}
|
||||
return "comparison_" + reportRunID, nil
|
||||
}
|
||||
|
||||
// DefaultDirectoryName derives the bundle directory name for a resolved report
|
||||
// output filename.
|
||||
func DefaultDirectoryName(reportOutputName string) (string, error) {
|
||||
if !strings.HasSuffix(reportOutputName, ".md") {
|
||||
return "", fmt.Errorf("report output name %q must end in .md", reportOutputName)
|
||||
}
|
||||
if !isArtifactBasename(reportOutputName) {
|
||||
return "", fmt.Errorf("report output name %q must be a basename", reportOutputName)
|
||||
}
|
||||
stem := strings.TrimSuffix(reportOutputName, ".md")
|
||||
if stem == "" {
|
||||
return "", fmt.Errorf("report output name %q has an empty stem", reportOutputName)
|
||||
}
|
||||
return "comparison-" + stem, nil
|
||||
}
|
||||
|
||||
// SHA256 returns a lowercase hexadecimal SHA-256 digest.
|
||||
func SHA256(content []byte) string {
|
||||
digest := sha256.Sum256(content)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
// NewSafeError returns bounded, valid UTF-8 error information suitable for a
|
||||
// comparison manifest.
|
||||
func NewSafeError(category, message string) SafeError {
|
||||
return SafeError{Category: category, Message: TruncateErrorMessage(message)}
|
||||
}
|
||||
|
||||
// TruncateErrorMessage returns a valid UTF-8 message of at most 1,024 bytes.
|
||||
func TruncateErrorMessage(message string) string {
|
||||
message = strings.ToValidUTF8(message, "\uFFFD")
|
||||
if len(message) <= maxErrorMessageBytes {
|
||||
return message
|
||||
}
|
||||
|
||||
end := maxErrorMessageBytes
|
||||
for end > 0 && !utf8.RuneStart(message[end]) {
|
||||
end--
|
||||
}
|
||||
return message[:end]
|
||||
}
|
||||
|
||||
// Validate checks every invariant required for a current comparison manifest.
|
||||
func (manifest Manifest) Validate() error {
|
||||
if manifest.SchemaVersion != SchemaVersion {
|
||||
return fmt.Errorf("unsupported comparison schema version %q", manifest.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(manifest.ComparisonID) == "" {
|
||||
return fmt.Errorf("comparison ID must not be blank")
|
||||
}
|
||||
if manifest.StartedAt.IsZero() || manifest.FinishedAt.IsZero() {
|
||||
return fmt.Errorf("comparison timestamps must be nonzero")
|
||||
}
|
||||
if manifest.StartedAt.Location() != time.UTC || manifest.FinishedAt.Location() != time.UTC {
|
||||
return fmt.Errorf("comparison timestamps must use UTC")
|
||||
}
|
||||
if manifest.FinishedAt.Before(manifest.StartedAt) {
|
||||
return fmt.Errorf("comparison finish time precedes start time")
|
||||
}
|
||||
if manifest.ValidPeriod.Start.IsZero() || manifest.ValidPeriod.End.IsZero() || !manifest.ValidPeriod.End.After(manifest.ValidPeriod.Start) {
|
||||
return fmt.Errorf("valid period must have a nonempty increasing range")
|
||||
}
|
||||
if strings.TrimSpace(manifest.ReportID) == "" || strings.TrimSpace(manifest.Timezone) == "" {
|
||||
return fmt.Errorf("report ID and timezone must not be blank")
|
||||
}
|
||||
if strings.TrimSpace(manifest.PromptID) == "" || strings.TrimSpace(manifest.PromptVersion) == "" || !isSHA256(manifest.PromptHash) {
|
||||
return fmt.Errorf("prompt identity is invalid")
|
||||
}
|
||||
if manifest.DataPackage.Path != DataPackageFilename || !isSHA256(manifest.DataPackage.SHA256) {
|
||||
return fmt.Errorf("data package reference is invalid")
|
||||
}
|
||||
if manifest.Total < 2 || manifest.Total != len(manifest.Results) {
|
||||
return fmt.Errorf("comparison result count is invalid")
|
||||
}
|
||||
if manifest.Succeeded < 0 || manifest.Failed < 0 || manifest.Succeeded+manifest.Failed != manifest.Total {
|
||||
return fmt.Errorf("comparison result totals are inconsistent")
|
||||
}
|
||||
|
||||
profiles := make(map[string]struct{}, len(manifest.Results))
|
||||
reportPaths := make(map[string]struct{}, len(manifest.Results))
|
||||
succeeded, failed := 0, 0
|
||||
for index, result := range manifest.Results {
|
||||
if result.Position != index+1 {
|
||||
return fmt.Errorf("result position %d is not ordered", result.Position)
|
||||
}
|
||||
if strings.TrimSpace(result.ProfileID) == "" {
|
||||
return fmt.Errorf("result %d has a blank profile ID", result.Position)
|
||||
}
|
||||
if _, duplicate := profiles[result.ProfileID]; duplicate {
|
||||
return fmt.Errorf("result %d duplicates profile ID %q", result.Position, result.ProfileID)
|
||||
}
|
||||
profiles[result.ProfileID] = struct{}{}
|
||||
if strings.TrimSpace(result.ModelName) == "" {
|
||||
return fmt.Errorf("result %d has a blank model name", result.Position)
|
||||
}
|
||||
|
||||
switch result.Status {
|
||||
case StatusSucceeded:
|
||||
succeeded++
|
||||
if result.ValidationStatus != "passed" {
|
||||
return fmt.Errorf("successful result %d did not pass validation", result.Position)
|
||||
}
|
||||
if result.Error != nil || !isReportPath(result.ReportPath) {
|
||||
return fmt.Errorf("successful result %d has invalid report details", result.Position)
|
||||
}
|
||||
if _, duplicate := reportPaths[result.ReportPath]; duplicate {
|
||||
return fmt.Errorf("result %d duplicates report path %q", result.Position, result.ReportPath)
|
||||
}
|
||||
reportPaths[result.ReportPath] = struct{}{}
|
||||
case StatusFailed:
|
||||
failed++
|
||||
if result.ReportPath != "" || result.Error == nil || !isValidationStatus(result.ValidationStatus) {
|
||||
return fmt.Errorf("failed result %d has invalid failure details", result.Position)
|
||||
}
|
||||
if err := result.Error.validate(); err != nil {
|
||||
return fmt.Errorf("failed result %d: %w", result.Position, err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("result %d has unsupported status %q", result.Position, result.Status)
|
||||
}
|
||||
}
|
||||
if succeeded != manifest.Succeeded || failed != manifest.Failed {
|
||||
return fmt.Errorf("comparison status counts do not match results")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (safeError SafeError) validate() error {
|
||||
if strings.TrimSpace(safeError.Category) == "" || strings.TrimSpace(safeError.Message) == "" {
|
||||
return fmt.Errorf("safe error category and message must not be blank")
|
||||
}
|
||||
if !utf8.ValidString(safeError.Message) || len(safeError.Message) > maxErrorMessageBytes {
|
||||
return fmt.Errorf("safe error message is not bounded UTF-8")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isValidationStatus(status string) bool {
|
||||
return status == "" || status == "failed" || status == "skipped"
|
||||
}
|
||||
|
||||
func isSHA256(value string) bool {
|
||||
if len(value) != sha256.Size*2 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isReportPath(value string) bool {
|
||||
return strings.HasSuffix(value, ".md") && isArtifactBasename(value) && value != ManifestFilename && value != DataPackageFilename
|
||||
}
|
||||
|
||||
func isArtifactBasename(value string) bool {
|
||||
return value != "" && value != "." && value != ".." &&
|
||||
!strings.ContainsAny(value, "/\\") && path.Base(value) == value
|
||||
}
|
||||
|
||||
// Validate checks that the manifest and in-memory bundle payloads agree.
|
||||
func (bundle LogicalBundle) Validate() error {
|
||||
if err := bundle.Manifest.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if SHA256(bundle.DataPackage) != bundle.Manifest.DataPackage.SHA256 {
|
||||
return fmt.Errorf("data package digest does not match manifest")
|
||||
}
|
||||
|
||||
if len(bundle.Reports) != bundle.Manifest.Succeeded {
|
||||
return fmt.Errorf("bundle report count does not match manifest")
|
||||
}
|
||||
reportIndex := 0
|
||||
for _, result := range bundle.Manifest.Results {
|
||||
if result.Status != StatusSucceeded {
|
||||
continue
|
||||
}
|
||||
report := bundle.Reports[reportIndex]
|
||||
if report.Position != result.Position || report.Path != result.ReportPath || !isReportPath(report.Path) {
|
||||
return fmt.Errorf("bundle report for result %d does not match manifest", result.Position)
|
||||
}
|
||||
reportIndex++
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeManifest validates and deterministically encodes a manifest as the
|
||||
// canonical two-space-indented JSON document with one trailing newline.
|
||||
func EncodeManifest(manifest Manifest) ([]byte, error) {
|
||||
if err := manifest.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoded, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode comparison manifest: %w", err)
|
||||
}
|
||||
return append(encoded, '\n'), nil
|
||||
}
|
||||
314
internal/comparison/comparison_test.go
Normal file
314
internal/comparison/comparison_test.go
Normal file
@@ -0,0 +1,314 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestValidateProfileIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
profileIDs []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "accepts exact case distinct IDs", profileIDs: []string{"weather-light", "Weather-Light"}},
|
||||
{name: "preserves surrounding whitespace", profileIDs: []string{" weather-light", "weather-deep "}},
|
||||
{name: "rejects one ID", profileIDs: []string{"weather-light"}, wantErr: true},
|
||||
{name: "rejects blank ID", profileIDs: []string{"weather-light", " \t\n"}, wantErr: true},
|
||||
{name: "rejects exact duplicate", profileIDs: []string{"weather-light", "weather-light"}, wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := ValidateProfileIDs(test.profileIDs)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("ValidateProfileIDs(%q) error = %v, want error %t", test.profileIDs, err, test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileSlug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
long := strings.Repeat("a", 62) + "-_more"
|
||||
tests := []struct {
|
||||
profileID string
|
||||
want string
|
||||
}{
|
||||
{profileID: "weather-light", want: "weather-light"},
|
||||
{profileID: "model.v1", want: "model-v1"},
|
||||
{profileID: "nested/path\\name", want: "nested-path-name"},
|
||||
{profileID: " weather\tlight\n", want: "weather-light"},
|
||||
{profileID: "\x00\x01", want: "profile"},
|
||||
{profileID: "météo-東京", want: "m-t-o"},
|
||||
{profileID: "___", want: "profile"},
|
||||
{profileID: long, want: strings.Repeat("a", 62)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.profileID, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := ProfileSlug(test.profileID); got != test.want {
|
||||
t.Fatalf("ProfileSlug(%q) = %q, want %q", test.profileID, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportNaming(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := OrdinalWidth(99); got != 2 {
|
||||
t.Fatalf("OrdinalWidth(99) = %d, want 2", got)
|
||||
}
|
||||
if got := OrdinalWidth(100); got != 3 {
|
||||
t.Fatalf("OrdinalWidth(100) = %d, want 3", got)
|
||||
}
|
||||
if got, err := ReportFilename(1, 3, "weather.light"); err != nil || got != "01-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "01-weather-light.md")
|
||||
}
|
||||
if got, err := ReportFilename(100, 100, "weather-light"); err != nil || got != "100-weather-light.md" {
|
||||
t.Fatalf("ReportFilename() = %q, %v, want %q, nil", got, err, "100-weather-light.md")
|
||||
}
|
||||
first, err := ReportFilename(1, 2, "model.v1")
|
||||
if err != nil {
|
||||
t.Fatalf("first ReportFilename() error = %v", err)
|
||||
}
|
||||
second, err := ReportFilename(2, 2, "model/v1")
|
||||
if err != nil {
|
||||
t.Fatalf("second ReportFilename() error = %v", err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("normalized profile collisions produced the same filename %q", first)
|
||||
}
|
||||
if _, err := ReportFilename(0, 2, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted position zero")
|
||||
}
|
||||
if _, err := ReportFilename(1, 0, "weather-light"); err == nil {
|
||||
t.Fatal("ReportFilename accepted zero profile count")
|
||||
}
|
||||
if got, err := BuildComparisonID("daily-2026-08-24"); err != nil || got != "comparison_daily-2026-08-24" {
|
||||
t.Fatalf("BuildComparisonID() = %q, %v", got, err)
|
||||
}
|
||||
if _, err := BuildComparisonID(" \t"); err == nil {
|
||||
t.Fatal("BuildComparisonID accepted blank run ID")
|
||||
}
|
||||
if got, err := DefaultDirectoryName("daily-2026-08-24.md"); err != nil || got != "comparison-daily-2026-08-24" {
|
||||
t.Fatalf("DefaultDirectoryName() = %q, %v", got, err)
|
||||
}
|
||||
for _, name := range []string{"daily.txt", "nested/daily.md", ".md"} {
|
||||
if _, err := DefaultDirectoryName(name); err == nil {
|
||||
t.Fatalf("DefaultDirectoryName(%q) accepted invalid name", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestEncodingAndRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := validManifest()
|
||||
encoded, err := EncodeManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeManifest() error = %v", err)
|
||||
}
|
||||
want := "{\n" +
|
||||
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
|
||||
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
|
||||
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
|
||||
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
|
||||
" \"reportId\": \"daily\",\n" +
|
||||
" \"validPeriod\": {\n" +
|
||||
" \"start\": \"2026-08-24T00:00:00-04:00\",\n" +
|
||||
" \"end\": \"2026-08-25T00:00:00-04:00\"\n" +
|
||||
" },\n" +
|
||||
" \"timezone\": \"America/New_York\",\n" +
|
||||
" \"promptId\": \"daily-report\",\n" +
|
||||
" \"promptVersion\": \"2026-08-01\",\n" +
|
||||
" \"promptHash\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n" +
|
||||
" \"dataPackage\": {\n" +
|
||||
" \"path\": \"data-package.yml\",\n" +
|
||||
" \"sha256\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n" +
|
||||
" },\n" +
|
||||
" \"total\": 2,\n" +
|
||||
" \"succeeded\": 1,\n" +
|
||||
" \"failed\": 1,\n" +
|
||||
" \"results\": [\n" +
|
||||
" {\n" +
|
||||
" \"position\": 1,\n" +
|
||||
" \"profileId\": \"weather-light\",\n" +
|
||||
" \"backendId\": \"openai\",\n" +
|
||||
" \"modelName\": \"gpt-5-mini\",\n" +
|
||||
" \"status\": \"succeeded\",\n" +
|
||||
" \"validationStatus\": \"passed\",\n" +
|
||||
" \"reportPath\": \"01-weather-light.md\"\n" +
|
||||
" },\n" +
|
||||
" {\n" +
|
||||
" \"position\": 2,\n" +
|
||||
" \"profileId\": \"weather-deep\",\n" +
|
||||
" \"modelName\": \"gpt-5\",\n" +
|
||||
" \"status\": \"failed\",\n" +
|
||||
" \"error\": {\n" +
|
||||
" \"category\": \"application\",\n" +
|
||||
" \"message\": \"generated text was rejected\"\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" ]\n" +
|
||||
"}\n"
|
||||
if string(encoded) != want {
|
||||
t.Fatalf("EncodeManifest() =\n%s\nwant\n%s", encoded, want)
|
||||
}
|
||||
|
||||
var decoded Manifest
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if err := decoded.Validate(); err != nil {
|
||||
t.Fatalf("decoded manifest validation error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Manifest)
|
||||
}{
|
||||
{name: "schema version", mutate: func(manifest *Manifest) { manifest.SchemaVersion = "v0" }},
|
||||
{name: "non UTC timestamp", mutate: func(manifest *Manifest) { manifest.StartedAt = manifest.StartedAt.In(time.FixedZone("UTC", 0)) }},
|
||||
{name: "reversed timestamps", mutate: func(manifest *Manifest) { manifest.FinishedAt = manifest.StartedAt.Add(-time.Second) }},
|
||||
{name: "empty valid period", mutate: func(manifest *Manifest) { manifest.ValidPeriod.End = manifest.ValidPeriod.Start }},
|
||||
{name: "bad prompt hash", mutate: func(manifest *Manifest) { manifest.PromptHash = "ABC" }},
|
||||
{name: "bad data package path", mutate: func(manifest *Manifest) { manifest.DataPackage.Path = "nested/data-package.yml" }},
|
||||
{name: "inconsistent totals", mutate: func(manifest *Manifest) { manifest.Succeeded = 2 }},
|
||||
{name: "unordered position", mutate: func(manifest *Manifest) { manifest.Results[1].Position = 3 }},
|
||||
{name: "duplicate profile", mutate: func(manifest *Manifest) { manifest.Results[1].ProfileID = manifest.Results[0].ProfileID }},
|
||||
{name: "unsupported status", mutate: func(manifest *Manifest) { manifest.Results[1].Status = "skipped" }},
|
||||
{name: "successful result without report", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "" }},
|
||||
{name: "successful result with error", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[0].Error = &SafeError{Category: "application", Message: "bad"}
|
||||
}},
|
||||
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
||||
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
||||
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
||||
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
||||
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
||||
manifest.Succeeded = 2
|
||||
manifest.Failed = 0
|
||||
}},
|
||||
{name: "oversized error", mutate: func(manifest *Manifest) { manifest.Results[1].Error.Message = strings.Repeat("x", 1025) }},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
manifest := validManifest()
|
||||
test.mutate(&manifest)
|
||||
if err := manifest.Validate(); err == nil {
|
||||
t.Fatal("Manifest.Validate() succeeded for invalid manifest")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalBundleValidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dataPackage := []byte("report: daily\n")
|
||||
manifest := validManifest()
|
||||
manifest.DataPackage.SHA256 = SHA256(dataPackage)
|
||||
bundle := LogicalBundle{
|
||||
Manifest: manifest,
|
||||
DataPackage: dataPackage,
|
||||
Reports: []BundleReport{{
|
||||
Position: 1,
|
||||
Path: "01-weather-light.md",
|
||||
Markdown: []byte("# Daily\n"),
|
||||
}},
|
||||
}
|
||||
if err := bundle.Validate(); err != nil {
|
||||
t.Fatalf("LogicalBundle.Validate() error = %v", err)
|
||||
}
|
||||
|
||||
bundle.Reports[0].Path = "other.md"
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted mismatched report path")
|
||||
}
|
||||
bundle.Reports[0].Path = "01-weather-light.md"
|
||||
bundle.Reports[0].Position = 2
|
||||
if err := bundle.Validate(); err == nil {
|
||||
t.Fatal("LogicalBundle.Validate() accepted unordered report position")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSHA256AndTruncateErrorMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got, want := SHA256([]byte("weather")), "e5e72beb4e3c6926d3dc9e3e2ef7833ba50cd919c2460a782b244fd071e920de"; got != want {
|
||||
t.Fatalf("SHA256() = %q, want %q", got, want)
|
||||
}
|
||||
message := strings.Repeat("€", 400)
|
||||
got := TruncateErrorMessage(message)
|
||||
if len(got) > 1024 || !utf8.ValidString(got) {
|
||||
t.Fatalf("TruncateErrorMessage() returned %d bytes of valid UTF-8 = %t", len(got), utf8.ValidString(got))
|
||||
}
|
||||
if want := strings.Repeat("€", 341); got != want {
|
||||
t.Fatalf("TruncateErrorMessage() = %q, want %q", got, want)
|
||||
}
|
||||
invalid := string([]byte{'x', 0xff, 'y'})
|
||||
if got := TruncateErrorMessage(invalid); !utf8.ValidString(got) {
|
||||
t.Fatal("TruncateErrorMessage() retained invalid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
func validManifest() Manifest {
|
||||
newYork := time.FixedZone("-0400", -4*60*60)
|
||||
return Manifest{
|
||||
SchemaVersion: SchemaVersion,
|
||||
ComparisonID: "comparison_daily-2026-08-24",
|
||||
StartedAt: time.Date(2026, time.August, 24, 12, 0, 0, 0, time.UTC),
|
||||
FinishedAt: time.Date(2026, time.August, 24, 12, 1, 0, 0, time.UTC),
|
||||
ReportID: "daily",
|
||||
ValidPeriod: ValidPeriod{
|
||||
Start: time.Date(2026, time.August, 24, 0, 0, 0, 0, newYork),
|
||||
End: time.Date(2026, time.August, 25, 0, 0, 0, 0, newYork),
|
||||
},
|
||||
Timezone: "America/New_York",
|
||||
PromptID: "daily-report",
|
||||
PromptVersion: "2026-08-01",
|
||||
PromptHash: strings.Repeat("a", 64),
|
||||
DataPackage: DataPackageReference{
|
||||
Path: DataPackageFilename,
|
||||
SHA256: strings.Repeat("a", 64),
|
||||
},
|
||||
Total: 2,
|
||||
Succeeded: 1,
|
||||
Failed: 1,
|
||||
Results: []Result{
|
||||
{
|
||||
Position: 1,
|
||||
ProfileID: "weather-light",
|
||||
BackendID: "openai",
|
||||
ModelName: "gpt-5-mini",
|
||||
Status: StatusSucceeded,
|
||||
ValidationStatus: "passed",
|
||||
ReportPath: "01-weather-light.md",
|
||||
},
|
||||
{
|
||||
Position: 2,
|
||||
ProfileID: "weather-deep",
|
||||
ModelName: "gpt-5",
|
||||
Status: StatusFailed,
|
||||
Error: &SafeError{Category: "application", Message: "generated text was rejected"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user