Harden PromptKit upgrade integration
This commit is contained in:
@@ -331,7 +331,7 @@ func (safeError SafeError) validate() error {
|
||||
}
|
||||
|
||||
func isValidationStatus(status string) bool {
|
||||
return status == "" || status == "failed" || status == "skipped"
|
||||
return status == "" || status == "passed" || status == "failed" || status == "skipped"
|
||||
}
|
||||
|
||||
func isSHA256(value string) bool {
|
||||
|
||||
@@ -202,6 +202,9 @@ func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||
{name: "successful result without passed validation", mutate: func(manifest *Manifest) { manifest.Results[0].ValidationStatus = "failed" }},
|
||||
{name: "failed result with report", mutate: func(manifest *Manifest) { manifest.Results[1].ReportPath = "02-weather-deep.md" }},
|
||||
{name: "failed result without error", mutate: func(manifest *Manifest) { manifest.Results[1].Error = nil }},
|
||||
{name: "failed passed result without repair provenance", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[1].ValidationStatus = "passed"
|
||||
}},
|
||||
{name: "traversal report path", mutate: func(manifest *Manifest) { manifest.Results[0].ReportPath = "../report.md" }},
|
||||
{name: "duplicate report path", mutate: func(manifest *Manifest) {
|
||||
manifest.Results[1] = Result{Position: 2, ProfileID: "weather-deep", ModelName: "gpt-5", Status: StatusSucceeded, ValidationStatus: "passed", ReportPath: manifest.Results[0].ReportPath}
|
||||
@@ -223,6 +226,17 @@ func TestManifestValidateRejectsInvariants(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestValidateAcceptsPostValidationFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
manifest := validManifest()
|
||||
manifest.Results[1].ValidationStatus = "passed"
|
||||
manifest.Results[1].RepairAttempts = intPtr(0)
|
||||
if err := manifest.Validate(); err != nil {
|
||||
t.Fatalf("Manifest.Validate() rejected post-validation failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalBundleValidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -519,8 +519,15 @@ func validateSafeErrorJSON(decoder *json.Decoder) error {
|
||||
}
|
||||
|
||||
func validateRepairAttemptsJSON(decoder *json.Decoder) error {
|
||||
var raw json.RawMessage
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
|
||||
return fmt.Errorf("repairAttempts must be an integer")
|
||||
}
|
||||
var value int
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return err
|
||||
}
|
||||
if value < 0 {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package comparison
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -146,6 +147,23 @@ func TestRecognizeBundleRejectsUnsafeAndMalformedContents(t *testing.T) {
|
||||
t.Helper()
|
||||
appendManifestField(t, directory, `"SchemaVersion": "weatherreporter.comparison.v1"`)
|
||||
}},
|
||||
{name: "null repair attempts", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := []byte(" \"status\": \"failed\",\n \"error\": {")
|
||||
after := []byte(" \"status\": \"failed\",\n \"repairAttempts\": null,\n \"error\": {")
|
||||
data = bytes.Replace(data, before, after, 1)
|
||||
if bytes.Equal(data, readFile(t, path)) {
|
||||
t.Fatal("failed to add null repairAttempts")
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "traversal report path", mutate: func(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
@@ -202,6 +220,30 @@ func TestRecognizeBundleRejectsUnsafeAndMalformedContents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRepairAttemptsJSONRejectsNonIntegerValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, value := range []string{"null", "-1", "1.5", `"1"`, "9223372036854775808", "{"} {
|
||||
value := value
|
||||
t.Run(value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := validateRepairAttemptsJSON(json.NewDecoder(strings.NewReader(value))); err == nil {
|
||||
t.Fatalf("validateRepairAttemptsJSON(%q) error = nil", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRepairAttemptsJSONAcceptsNonnegativeIntegers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, value := range []string{"0", "1"} {
|
||||
if err := validateRepairAttemptsJSON(json.NewDecoder(strings.NewReader(value))); err != nil {
|
||||
t.Fatalf("validateRepairAttemptsJSON(%q) error = %v", value, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func appendManifestField(t *testing.T, directory, field string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(directory, ManifestFilename)
|
||||
|
||||
Reference in New Issue
Block a user