eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Bound generated text content and diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-13 01:47:47 +00:00
parent f8beed04cf
commit e520ffb13b
20 changed files with 400 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/generatedtext/daily_test.go
View File

@@ -62,7 +62,7 @@ func TestValidateDailyRejectsInvalidInput(t *testing.T) {
{
name: "unknown field",
in: `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"extra":"value"}`,
want: `unknown field "extra"`,
want: "unsupported field",
},
{
name: "missing summary",
@@ -87,7 +87,7 @@ func TestValidateDailyRejectsInvalidInput(t *testing.T) {
{
name: "forecast discussion wrong type",
in: `{"summary":"Showers are possible during the selected day.","forecast_discussion":"A front will keep rain chances in the forecast."}`,
want: "cannot unmarshal string",
want: "forecast discussion must be an array",
},
{
name: "multiple values",

6
internal/generatedtext/day_style_test.go
View File

@@ -109,14 +109,14 @@ func TestValidateDayStyleGeneratedTextSharedBehavior(t *testing.T) {
if err == nil {
t.Fatal("validate() error = nil, want error")
}
if !strings.Contains(err.Error(), `unknown field "extra"`) {
t.Fatalf("validate() error = %v, want unknown field error", err)
if !strings.Contains(err.Error(), "unsupported field") {
t.Fatalf("validate() error = %v, want unsupported field error", err)
}
})
t.Run("rejects retired confidence field", func(t *testing.T) {
_, _, err := report.validate([]byte(`{"summary":"Shared summary.","forecast_discussion":["First paragraph."],"precipitation_timing":"","confidence":"Medium"}`))
if err == nil || !strings.Contains(err.Error(), `unknown field "confidence"`) {
if err == nil || !strings.Contains(err.Error(), "unsupported field") {
t.Fatalf("validate() error = %v, want retired confidence field rejection", err)
}
})

8
internal/generatedtext/hourly_test.go
View File

@@ -58,7 +58,7 @@ func TestValidateHourlyRejectsInvalidInput(t *testing.T) {
{
name: "unknown field",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","extra":"value"}`,
want: `unknown field "extra"`,
want: "unsupported field",
},
{
name: "missing summary",
@@ -83,17 +83,17 @@ func TestValidateHourlyRejectsInvalidInput(t *testing.T) {
{
name: "retired confidence field rejected",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"","confidence":"Medium"}`,
want: `unknown field "confidence"`,
want: "unsupported field",
},
{
name: "old timing field rejected",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","timing":"Late morning."}`,
want: `unknown field "timing"`,
want: "unsupported field",
},
{
name: "old impacts field rejected",
in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","impacts":"Brief downpours."}`,
want: `unknown field "impacts"`,
want: "unsupported field",
},
{
name: "multiple values",

30
internal/generatedtext/json.go
View File

@@ -16,18 +16,21 @@ var generatedTextFieldNames = map[string]struct{}{
func decodeGeneratedText[T any](data []byte, name string) (T, error) {
var value T
if err := validateGeneratedTextSize(data, name); err != nil {
return value, err
}
if err := validateGeneratedTextObject(data, name); err != nil {
return value, err
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&value); err != nil {
return value, fmt.Errorf("decode %s generated text: %w", name, err)
return value, fmt.Errorf("decode %s generated text: invalid field value", name)
}
var extra any
if err := decoder.Decode(&extra); err != nil {
if err != io.EOF {
return value, fmt.Errorf("decode %s generated text: %w", name, err)
return value, fmt.Errorf("decode %s generated text: invalid JSON", name)
}
return value, nil
}
@@ -38,35 +41,39 @@ func validateGeneratedTextObject(data []byte, name string) error {
decoder := json.NewDecoder(bytes.NewReader(data))
token, err := decoder.Token()
if err != nil {
return fmt.Errorf("decode %s generated text: %w", name, err)
return fmt.Errorf("decode %s generated text: invalid JSON", name)
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' {
return fmt.Errorf("decode %s generated text: expected JSON object", name)
}
seen := make(map[string]struct{}, len(generatedTextFieldNames))
totalCharacters := 0
for decoder.More() {
token, err := decoder.Token()
if err != nil {
return fmt.Errorf("decode %s generated text: %w", name, err)
return fmt.Errorf("decode %s generated text: invalid JSON", name)
}
field, ok := token.(string)
if !ok {
return fmt.Errorf("decode %s generated text: expected object field name", name)
}
if _, ok := generatedTextFieldNames[field]; !ok {
return fmt.Errorf("decode %s generated text: json: unknown field %q", name, field)
return fmt.Errorf("decode %s generated text: unsupported field", name)
}
if _, ok := seen[field]; ok {
return fmt.Errorf("decode %s generated text: duplicate field %q", name, field)
return fmt.Errorf("decode %s generated text: duplicate %s field", name, generatedTextFieldLabel(field))
}
seen[field] = struct{}{}
var value json.RawMessage
if err := decoder.Decode(&value); err != nil {
return fmt.Errorf("decode %s generated text: %w", name, err)
return fmt.Errorf("decode %s generated text: invalid JSON", name)
}
if err := validateGeneratedTextField(name, field, value, &totalCharacters); err != nil {
return err
}
}
if _, err := decoder.Token(); err != nil {
return fmt.Errorf("decode %s generated text: %w", name, err)
return fmt.Errorf("decode %s generated text: invalid JSON", name)
}
return nil
}
@@ -74,15 +81,12 @@ func validateGeneratedTextObject(data []byte, name string) error {
func requireGeneratedTextStringField(data []byte, name, field string) error {
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
return fmt.Errorf("decode %s generated text: %w", name, err)
return fmt.Errorf("decode %s generated text: invalid JSON", name)
}
raw, ok := fields[field]
_, ok := fields[field]
if !ok {
return fmt.Errorf("%s generated text %s is required", name, strings.ReplaceAll(field, "_", " "))
}
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
return fmt.Errorf("%s generated text %s must be a string", name, strings.ReplaceAll(field, "_", " "))
}
return nil
}

50
internal/generatedtext/json_test.go
View File

@@ -87,6 +87,56 @@ func TestTypedValidatorsMatchEmbeddedSchemaObjectShape(t *testing.T) {
}
}
func TestTypedValidatorsMatchEmbeddedSchemaContentLimits(t *testing.T) {
validators := []struct {
name string
valid string
invalid []string
validate func([]byte) error
}{
{
name: "daily",
valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)),
invalid: []string{fmt.Sprintf(`{"summary":%q,"forecast_discussion":["discussion"],"precipitation_timing":""}`, strings.Repeat("s", 4_001)), fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"discussion",`, 13), ",")), `{"summary":"summary","forecast_discussion":[null],"precipitation_timing":""}`},
validate: validateDailyJSON,
},
{
name: "today",
valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)),
invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("d", 4_001))},
validate: validateTodayJSON,
},
{
name: "tomorrow",
valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)),
invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":["discussion"],"precipitation_timing":%q}`, strings.Repeat("t", 4_001))},
validate: validateTomorrowJSON,
},
{
name: "hourly",
valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":%q,"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 12_000)),
invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":%q,"precipitation_timing":""}`, strings.Repeat("d", 12_001))},
validate: validateHourlyJSON,
},
}
for _, validator := range validators {
t.Run(validator.name, func(t *testing.T) {
schema := generatedTextSchema(t, validator.name)
for _, input := range append([]string{validator.valid}, validator.invalid...) {
instance, err := jsonschema.UnmarshalJSON(strings.NewReader(input))
if err != nil {
t.Fatalf("parse schema instance: %v", err)
}
schemaValid := schema.Validate(instance) == nil
typedValid := validator.validate([]byte(input)) == nil
if schemaValid != typedValid {
t.Fatalf("schema valid = %t, typed valid = %t", schemaValid, typedValid)
}
}
})
}
}
func generatedTextSchema(t *testing.T, id string) *jsonschema.Schema {
t.Helper()
data, err := promptassets.Schema(id)

120
internal/generatedtext/limits.go Normal file
View File

@@ -0,0 +1,120 @@
package generatedtext
import (
"bytes"
"encoding/json"
"fmt"
"unicode/utf8"
)
const (
// MaxGeneratedTextBytes is the largest provider response accepted for one
// generated-text report before JSON decoding begins.
MaxGeneratedTextBytes = 64 * 1024
maxGeneratedTextCharacters = 20_000
maxGeneratedTextStringCharacters = 4_000
maxHourlyDiscussionCharacters = 12_000
maxDayStyleDiscussionParagraphs = 12
maxDayStyleDiscussionParagraphChars = 4_000
)
// ValidateRawOutput rejects generated JSON before it is decoded or retained.
func ValidateRawOutput(data []byte) error {
if len(data) > MaxGeneratedTextBytes {
return fmt.Errorf("generated text exceeds the %d-byte limit", MaxGeneratedTextBytes)
}
return nil
}
func validateGeneratedTextSize(data []byte, name string) error {
if err := ValidateRawOutput(data); err != nil {
return fmt.Errorf("%s %w", name, err)
}
return nil
}
func validateGeneratedTextField(name, field string, raw json.RawMessage, total *int) error {
switch field {
case "summary", "precipitation_timing":
return validateGeneratedTextString(name, field, raw, maxGeneratedTextStringCharacters, total)
case "forecast_discussion":
if name == "hourly" {
return validateGeneratedTextString(name, field, raw, maxHourlyDiscussionCharacters, total)
}
return validateDayStyleDiscussion(name, raw, total)
default:
return fmt.Errorf("decode %s generated text: unsupported field", name)
}
}
func validateGeneratedTextString(name, field string, raw json.RawMessage, limit int, total *int) error {
if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
return fmt.Errorf("%s generated text %s must be a string", name, generatedTextFieldLabel(field))
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
return fmt.Errorf("%s generated text %s must be a string", name, generatedTextFieldLabel(field))
}
characters := utf8.RuneCountInString(value)
if characters > limit {
return fmt.Errorf("%s generated text %s exceeds the %d-character limit", name, generatedTextFieldLabel(field), limit)
}
return addGeneratedTextCharacters(name, characters, total)
}
func validateDayStyleDiscussion(name string, raw json.RawMessage, total *int) error {
decoder := json.NewDecoder(bytes.NewReader(raw))
token, err := decoder.Token()
if err != nil {
return fmt.Errorf("%s generated text forecast discussion must be an array", name)
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '[' {
return fmt.Errorf("%s generated text forecast discussion must be an array", name)
}
paragraphs := 0
for decoder.More() {
paragraphs++
if paragraphs > maxDayStyleDiscussionParagraphs {
return fmt.Errorf("%s generated text forecast discussion exceeds the %d-paragraph limit", name, maxDayStyleDiscussionParagraphs)
}
var rawParagraph json.RawMessage
if err := decoder.Decode(&rawParagraph); err != nil || bytes.Equal(bytes.TrimSpace(rawParagraph), []byte("null")) {
return fmt.Errorf("%s generated text forecast discussion paragraphs must be strings", name)
}
var paragraph string
if err := json.Unmarshal(rawParagraph, &paragraph); err != nil {
return fmt.Errorf("%s generated text forecast discussion paragraphs must be strings", name)
}
characters := utf8.RuneCountInString(paragraph)
if characters > maxDayStyleDiscussionParagraphChars {
return fmt.Errorf("%s generated text forecast discussion paragraphs exceed the %d-character limit", name, maxDayStyleDiscussionParagraphChars)
}
if err := addGeneratedTextCharacters(name, characters, total); err != nil {
return err
}
}
if _, err := decoder.Token(); err != nil {
return fmt.Errorf("%s generated text forecast discussion must be an array", name)
}
return nil
}
func addGeneratedTextCharacters(name string, characters int, total *int) error {
*total += characters
if *total > maxGeneratedTextCharacters {
return fmt.Errorf("%s generated text exceeds the %d-character limit", name, maxGeneratedTextCharacters)
}
return nil
}
func generatedTextFieldLabel(field string) string {
if field == "forecast_discussion" {
return "forecast discussion"
}
if field == "precipitation_timing" {
return "precipitation timing"
}
return field
}

112
internal/generatedtext/limits_test.go Normal file
View File

@@ -0,0 +1,112 @@
package generatedtext
import (
"fmt"
"strings"
"testing"
)
func TestValidateHourlyGeneratedTextEnforcesContentLimits(t *testing.T) {
input := func(summary, discussion, timing string) []byte {
return []byte(fmt.Sprintf(`{"summary":%q,"forecast_discussion":%q,"precipitation_timing":%q}`, summary, discussion, timing))
}
for _, test := range []struct {
name string
input []byte
want string
}{
{name: "summary at limit", input: input(strings.Repeat("s", maxGeneratedTextStringCharacters), "discussion", "")},
{name: "summary over limit", input: input(strings.Repeat("s", maxGeneratedTextStringCharacters+1), "discussion", ""), want: "summary exceeds"},
{name: "discussion at limit", input: input("summary", strings.Repeat("d", maxHourlyDiscussionCharacters), "")},
{name: "discussion over limit", input: input("summary", strings.Repeat("d", maxHourlyDiscussionCharacters+1), ""), want: "forecast discussion exceeds"},
{name: "total at limit", input: input(strings.Repeat("s", 4_000), strings.Repeat("d", 12_000), strings.Repeat("t", 4_000))},
} {
t.Run(test.name, func(t *testing.T) {
err := validateHourlyJSON(test.input)
if test.want == "" && err != nil {
t.Fatalf("validate() error = %v", err)
}
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
t.Fatalf("validate() error = %v, want %q", err, test.want)
}
})
}
}
func TestValidateDayStyleGeneratedTextEnforcesDiscussionLimits(t *testing.T) {
for _, test := range []struct {
name string
input string
want string
}{
{
name: "paragraph count at limit",
input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"p",`, maxDayStyleDiscussionParagraphs), ",")),
},
{
name: "paragraph count over limit",
input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"p",`, maxDayStyleDiscussionParagraphs+1), ",")),
want: "12-paragraph limit",
},
{
name: "paragraph at limit",
input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("p", maxDayStyleDiscussionParagraphChars)),
},
{
name: "paragraph over limit",
input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("p", maxDayStyleDiscussionParagraphChars+1)),
want: "paragraphs exceed the 4000-character limit",
},
{
name: "total at limit",
input: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q,%q,%q,%q],"precipitation_timing":""}`,
strings.Repeat("s", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000)),
},
{
name: "total over limit",
input: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q,%q,%q,%q,%q],"precipitation_timing":""}`,
strings.Repeat("s", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000)),
want: "exceeds the 20000-character limit",
},
} {
t.Run(test.name, func(t *testing.T) {
err := validateDailyJSON([]byte(test.input))
if test.want == "" && err != nil {
t.Fatalf("validate() error = %v", err)
}
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
t.Fatalf("validate() error = %v, want %q", err, test.want)
}
})
}
}
func TestValidateGeneratedTextRejectsOversizedRawOutputWithoutLeakingContent(t *testing.T) {
valid := []byte(`{"summary":"summary","forecast_discussion":"discussion","precipitation_timing":""}`)
atLimit := append([]byte(strings.Repeat(" ", MaxGeneratedTextBytes-len(valid))), valid...)
if err := validateHourlyJSON(atLimit); err != nil {
t.Fatalf("validate() at raw limit error = %v", err)
}
marker := "provider-controlled-marker"
unknownPrefix := `{"summary":"summary","forecast_discussion":"discussion","precipitation_timing":"","`
unknownSuffix := `":"value"}`
unknownKey := marker + strings.Repeat("x", MaxGeneratedTextBytes-len(unknownPrefix)-len(unknownSuffix)-len(marker))
unknownField := []byte(unknownPrefix + unknownKey + unknownSuffix)
err := validateHourlyJSON(unknownField)
if err == nil || !strings.Contains(err.Error(), "unsupported field") {
t.Fatalf("validate() error = %v, want unsupported field error", err)
}
if len(err.Error()) > 128 || strings.Contains(err.Error(), marker) {
t.Fatalf("validate() leaked unknown field content: %q", err)
}
input := append(atLimit, []byte(marker)...)
err = validateHourlyJSON(input)
if err == nil || !strings.Contains(err.Error(), "65536-byte limit") {
t.Fatalf("validate() error = %v, want raw size error", err)
}
if len(err.Error()) > 128 || strings.Contains(err.Error(), marker) {
t.Fatalf("validate() leaked oversized provider content: %q", err)
}
}

4
internal/generatedtext/today_test.go
View File

@@ -62,7 +62,7 @@ func TestValidateTodayRejectsInvalidInput(t *testing.T) {
{
name: "unknown field",
in: `{"summary":"Showers are likely today.","forecast_discussion":["A front will keep rain chances elevated."],"extra":"value"}`,
want: `unknown field "extra"`,
want: "unsupported field",
},
{
name: "missing summary",
@@ -87,7 +87,7 @@ func TestValidateTodayRejectsInvalidInput(t *testing.T) {
{
name: "forecast discussion wrong type",
in: `{"summary":"Showers are likely today.","forecast_discussion":"A front will keep rain chances elevated."}`,
want: "cannot unmarshal string",
want: "forecast discussion must be an array",
},
{
name: "multiple values",

4
internal/generatedtext/tomorrow_test.go
View File

@@ -62,7 +62,7 @@ func TestValidateTomorrowRejectsInvalidInput(t *testing.T) {
{
name: "unknown field",
in: `{"summary":"Storms become more likely tomorrow.","forecast_discussion":["A front will keep showers in the forecast."],"extra":"value"}`,
want: `unknown field "extra"`,
want: "unsupported field",
},
{
name: "missing summary",
@@ -87,7 +87,7 @@ func TestValidateTomorrowRejectsInvalidInput(t *testing.T) {
{
name: "forecast discussion wrong type",
in: `{"summary":"Storms become more likely tomorrow.","forecast_discussion":"A front will keep showers in the forecast."}`,
want: "cannot unmarshal string",
want: "forecast discussion must be an array",
},
{
name: "multiple values",