Enforce generated text report identity

2026-08-13 01:36:56 +00:00
parent 44af91cadf
commit f8beed04cf
7 changed files with 251 additions and 17 deletions
--- a/internal/generatedtext/json_test.go
+++ b/internal/generatedtext/json_test.go
@@ -0,0 +1,130 @@
+package generatedtext
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/weatherreporter/internal/promptassets"
+	"github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+func TestGeneratedTextValidatorsRequireExactObjectFields(t *testing.T) {
+	validators := []struct {
+		name     string
+		valid    string
+		validate func([]byte) error
+	}{
+		{name: "daily", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateDailyJSON},
+		{name: "today", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateTodayJSON},
+		{name: "tomorrow", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateTomorrowJSON},
+		{name: "hourly", valid: `{"summary":"Summary","forecast_discussion":"Discussion","precipitation_timing":""}`, validate: validateHourlyJSON},
+	}
+	for _, validator := range validators {
+		t.Run(validator.name, func(t *testing.T) {
+			for _, test := range []struct {
+				name  string
+				input string
+				valid bool
+			}{
+				{name: "canonical", input: validator.valid, valid: true},
+				{name: "missing", input: strings.Replace(validator.valid, `"summary":"Summary",`, "", 1)},
+				{name: "additional", input: strings.Replace(validator.valid, "}", `,"extra":"value"}`, 1)},
+				{name: "case variant summary", input: strings.Replace(validator.valid, `"summary"`, `"Summary"`, 1)},
+				{name: "case variant discussion", input: strings.Replace(validator.valid, `"forecast_discussion"`, `"Forecast_Discussion"`, 1)},
+				{name: "duplicate", input: strings.Replace(validator.valid, `"summary":"Summary",`, `"summary":"Summary","summary":"Other",`, 1)},
+			} {
+				t.Run(test.name, func(t *testing.T) {
+					err := validator.validate([]byte(test.input))
+					if (err == nil) != test.valid {
+						t.Fatalf("validate(%s) error = %v, want valid = %t", test.input, err, test.valid)
+					}
+				})
+			}
+		})
+	}
+}
+
+func TestTypedValidatorsMatchEmbeddedSchemaObjectShape(t *testing.T) {
+	validators := []struct {
+		name     string
+		valid    string
+		validate func([]byte) error
+	}{
+		{name: "daily", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateDailyJSON},
+		{name: "today", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateTodayJSON},
+		{name: "tomorrow", valid: `{"summary":"Summary","forecast_discussion":["Discussion"],"precipitation_timing":""}`, validate: validateTomorrowJSON},
+		{name: "hourly", valid: `{"summary":"Summary","forecast_discussion":"Discussion","precipitation_timing":""}`, validate: validateHourlyJSON},
+	}
+	for _, validator := range validators {
+		t.Run(validator.name, func(t *testing.T) {
+			schema := generatedTextSchema(t, validator.name)
+			for _, test := range []struct {
+				name  string
+				input string
+			}{
+				{name: "canonical", input: validator.valid},
+				{name: "missing", input: strings.Replace(validator.valid, `"summary":"Summary",`, "", 1)},
+				{name: "additional", input: strings.Replace(validator.valid, "}", `,"extra":"value"}`, 1)},
+				{name: "case variant", input: strings.Replace(validator.valid, `"summary"`, `"Summary"`, 1)},
+				{name: "null", input: strings.Replace(validator.valid, `"precipitation_timing":""`, `"precipitation_timing":null`, 1)},
+				{name: "wrong type", input: strings.Replace(validator.valid, `"summary":"Summary"`, `"summary":false`, 1)},
+			} {
+				t.Run(test.name, func(t *testing.T) {
+					instance, err := jsonschema.UnmarshalJSON(strings.NewReader(test.input))
+					if err != nil {
+						t.Fatalf("parse schema instance: %v", err)
+					}
+					schemaValid := schema.Validate(instance) == nil
+					typedValid := validator.validate([]byte(test.input)) == nil
+					if schemaValid != typedValid {
+						t.Fatalf("schema valid = %t, typed valid = %t for %s", schemaValid, typedValid, test.input)
+					}
+				})
+			}
+		})
+	}
+}
+
+func generatedTextSchema(t *testing.T, id string) *jsonschema.Schema {
+	t.Helper()
+	data, err := promptassets.Schema(id)
+	if err != nil {
+		t.Fatalf("Schema(%q) error = %v", id, err)
+	}
+	document, err := jsonschema.UnmarshalJSON(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("parse schema %q: %v", id, err)
+	}
+	compiler := jsonschema.NewCompiler()
+	location := fmt.Sprintf("https://weatherreporter.example/schemas/%s.json", id)
+	if err := compiler.AddResource(location, document); err != nil {
+		t.Fatalf("add schema %q: %v", id, err)
+	}
+	schema, err := compiler.Compile(location)
+	if err != nil {
+		t.Fatalf("compile schema %q: %v", id, err)
+	}
+	return schema
+}
+
+func validateDailyJSON(data []byte) error {
+	_, _, err := ValidateDaily(data)
+	return err
+}
+
+func validateTodayJSON(data []byte) error {
+	_, _, err := ValidateToday(data)
+	return err
+}
+
+func validateTomorrowJSON(data []byte) error {
+	_, _, err := ValidateTomorrow(data)
+	return err
+}
+
+func validateHourlyJSON(data []byte) error {
+	_, _, err := ValidateHourly(data)
+	return err
+}