eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Migrate comparison bundles to v2

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 19:52:41 +00:00
parent 0c9cd6d5fb
commit b3b23fb381
9 changed files with 120 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/app/comparison.go
View File

@@ -59,6 +59,7 @@ type ComparisonProfileResult struct {
ModelName string
Status string
ValidationStatus promptexec.ValidationStatus
RepairAttempts *int
ReportPath string
LLMDebugPath string
Error *comparison.SafeError
@@ -197,6 +198,9 @@ func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfi
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
}
if outcome.RepairAttempts != nil {
profile.RepairAttempts = repairAttemptsPointer(*outcome.RepairAttempts)
}
if published && outcome.Status == comparison.StatusSucceeded {
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
}
@@ -226,6 +230,9 @@ func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []c
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
}
if outcome.RepairAttempts != nil {
manifestResult.RepairAttempts = repairAttemptsPointer(*outcome.RepairAttempts)
}
if outcome.Status == comparison.StatusSucceeded {
manifestResult.ReportPath = outcome.ReportPath
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})

9
internal/cli/result.go
View File

@@ -107,6 +107,7 @@ type comparisonProfileSummary struct {
ModelName string `json:"modelName"`
Status string `json:"status"`
ValidationStatus string `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"repairAttempts,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
Error *comparison.SafeError `json:"error,omitempty"`
@@ -167,6 +168,11 @@ func newGenerateNotificationSummary(result *app.NotificationResult) *generateNot
return summary
}
func repairAttemptsCopy(value int) *int {
copy := value
return &copy
}
func newBatchSummary(result *app.BatchResult, err error) batchSummary {
summary := batchSummary{Command: commandRun}
if result == nil {
@@ -208,6 +214,9 @@ func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSum
Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
})
if profile.RepairAttempts != nil {
summary.Results[len(summary.Results)-1].RepairAttempts = repairAttemptsCopy(*profile.RepairAttempts)
}
}
summary.Status = comparisonSummaryStatus(result, err)
if err != nil {

11
internal/comparison/comparison.go
View File

@@ -15,7 +15,7 @@ import (
const (
// SchemaVersion identifies the supported comparison manifest schema.
SchemaVersion = "weatherreporter.comparison.v1"
SchemaVersion = "weatherreporter.comparison.v2"
// ManifestFilename is the canonical name of a comparison manifest.
ManifestFilename = "comparison.json"
@@ -71,6 +71,7 @@ type Result struct {
ModelName string `json:"modelName"`
Status string `json:"status"`
ValidationStatus string `json:"validationStatus,omitempty"`
RepairAttempts *int `json:"repairAttempts,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
Error *SafeError `json:"error,omitempty"`
}
@@ -276,11 +277,14 @@ func (manifest Manifest) Validate() error {
if strings.TrimSpace(result.ModelName) == "" {
return fmt.Errorf("result %d has a blank model name", result.Position)
}
if result.RepairAttempts != nil && *result.RepairAttempts < 0 {
return fmt.Errorf("result %d has negative repair attempts", result.Position)
}
switch result.Status {
case StatusSucceeded:
succeeded++
if result.ValidationStatus != "passed" {
if result.ValidationStatus != "passed" || result.RepairAttempts == nil {
return fmt.Errorf("successful result %d did not pass validation", result.Position)
}
expectedPath, err := ReportFilename(result.Position, manifest.Total, result.ProfileID)
@@ -302,6 +306,9 @@ func (manifest Manifest) Validate() error {
if err := result.Error.validate(); err != nil {
return fmt.Errorf("failed result %d: %w", result.Position, err)
}
if result.ValidationStatus != "" && result.RepairAttempts == nil {
return fmt.Errorf("failed result %d has validation without repair provenance", result.Position)
}
default:
return fmt.Errorf("result %d has unsupported status %q", result.Position, result.Status)
}

9
internal/comparison/comparison_test.go
View File

@@ -118,7 +118,7 @@ func TestManifestEncodingAndRoundTrip(t *testing.T) {
t.Fatalf("EncodeManifest() error = %v", err)
}
want := "{\n" +
" \"schemaVersion\": \"weatherreporter.comparison.v1\",\n" +
" \"schemaVersion\": \"weatherreporter.comparison.v2\",\n" +
" \"comparisonId\": \"comparison_daily-2026-08-24\",\n" +
" \"startedAt\": \"2026-08-24T12:00:00Z\",\n" +
" \"finishedAt\": \"2026-08-24T12:01:00Z\",\n" +
@@ -146,6 +146,7 @@ func TestManifestEncodingAndRoundTrip(t *testing.T) {
" \"modelName\": \"gpt-5-mini\",\n" +
" \"status\": \"succeeded\",\n" +
" \"validationStatus\": \"passed\",\n" +
" \"repairAttempts\": 0,\n" +
" \"reportPath\": \"01-weather-light.md\"\n" +
" },\n" +
" {\n" +
@@ -309,6 +310,7 @@ func validManifest() Manifest {
ModelName: "gpt-5-mini",
Status: StatusSucceeded,
ValidationStatus: "passed",
RepairAttempts: intPtr(0),
ReportPath: "01-weather-light.md",
},
{
@@ -321,3 +323,8 @@ func validManifest() Manifest {
},
}
}
func intPtr(value int) *int {
copy := value
return &copy
}

73
internal/comparison/publish.go
View File

@@ -388,6 +388,7 @@ var resultFields = map[string]jsonValueValidator{
"modelName": nil,
"status": nil,
"validationStatus": nil,
"repairAttempts": validateRepairAttemptsJSON,
"reportPath": nil,
"error": validateSafeErrorJSON,
}
@@ -430,7 +431,7 @@ func validateResultsJSON(decoder *json.Decoder) error {
return fmt.Errorf("results must be an array")
}
for decoder.More() {
if err := validateJSONObject(decoder, resultFields); err != nil {
if err := validateOrderedJSONObject(decoder, resultFields, []string{"position", "profileId", "backendId", "modelName", "status", "validationStatus", "repairAttempts", "reportPath", "error"}); err != nil {
return err
}
}
@@ -444,6 +445,65 @@ func validateResultsJSON(decoder *json.Decoder) error {
return nil
}
func validateOrderedJSONObject(decoder *json.Decoder, fields map[string]jsonValueValidator, order []string) error {
token, err := decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' {
return fmt.Errorf("manifest value must be an object")
}
seen := make(map[string]struct{}, len(fields))
last := -1
for decoder.More() {
token, err := decoder.Token()
if err != nil {
return err
}
name, ok := token.(string)
if !ok {
return fmt.Errorf("manifest field name is invalid")
}
validator, known := fields[name]
if !known {
return fmt.Errorf("manifest field %q is not canonical", name)
}
if _, duplicate := seen[name]; duplicate {
return fmt.Errorf("manifest field %q is duplicated", name)
}
position := -1
for index, candidate := range order {
if candidate == name {
position = index
break
}
}
if position <= last {
return fmt.Errorf("manifest field %q is out of canonical order", name)
}
seen[name] = struct{}{}
last = position
if validator == nil {
var value json.RawMessage
if err := decoder.Decode(&value); err != nil {
return err
}
continue
}
if err := validator(decoder); err != nil {
return err
}
}
token, err = decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '}' {
return fmt.Errorf("manifest object has an invalid terminator")
}
return nil
}
func validateSafeErrorJSON(decoder *json.Decoder) error {
token, err := decoder.Token()
if err != nil {
@@ -458,6 +518,17 @@ func validateSafeErrorJSON(decoder *json.Decoder) error {
return validateJSONObjectBody(decoder, safeErrorFields)
}
func validateRepairAttemptsJSON(decoder *json.Decoder) error {
var value int
if err := decoder.Decode(&value); err != nil {
return err
}
if value < 0 {
return fmt.Errorf("repairAttempts must not be negative")
}
return nil
}
func validateJSONObject(decoder *json.Decoder, fields map[string]jsonValueValidator) error {
token, err := decoder.Token()
if err != nil {

2
internal/comparison/publish_test.go
View File

@@ -1077,7 +1077,7 @@ func testBundleWithReports(t *testing.T, count int) LogicalBundle {
}
manifest.Results[i] = Result{
Position: position, ProfileID: profileID, ModelName: "gpt-5-mini",
Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: path,
Status: StatusSucceeded, ValidationStatus: "passed", RepairAttempts: intPtr(0), ReportPath: path,
}
reports[i] = BundleReport{Position: position, Path: path, Markdown: []byte("# Daily\n")}
}