diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index fce02b0..b73fea7 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -51,6 +51,6 @@ its compatibility rules are defined by the [comparison bundle contract](comparison-bundle.md); the user-facing command contract is in the [CLI reference](../cli.md). -The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows. +The generated-text schemas require `summary`, `forecast_discussion`, and `precipitation_timing`, and reject additional properties. Promptkit results are accepted only when their raw JSON is at most 64 KiB; the adapter drops larger results before copying them into Weatherreporter's execution values or debug artifacts. The validator also limits total generated prose to 20,000 characters, with 4,000-character summary and timing fields, a 12,000-character Hourly discussion, and at most 12 day-style paragraphs of 4,000 characters each. Prompts return an empty string for `precipitation_timing` when the deterministic package contains no precipitation windows. Prompt/profile configuration and the maintained local override example are owned by the [configuration reference](../config.md). Adapter construction and mapping are documented in the [Promptkit adapter internals](../internal/promptkit-adapter.md). diff --git a/docs/internal/generatedtext.md b/docs/internal/generatedtext.md index 14b378a..6112bd8 100644 --- a/docs/internal/generatedtext.md +++ b/docs/internal/generatedtext.md @@ -24,6 +24,20 @@ prose to render. Typed decoding requires the exact lowercase JSON field names, rejects missing, duplicate, case-variant, and unknown fields, and checks field shapes; no general-purpose JSON Schema engine is used at runtime. +The validator accepts at most 64 KiB of raw JSON before it allocates typed +values. Its JSON Schemas and typed checks limit `summary` and +`precipitation_timing` to 4,000 characters each. Hourly +`forecast_discussion` is limited to 12,000 characters. Day-style discussion +accepts at most 12 paragraphs of at most 4,000 characters each. Across all +prose fields, one report may contain at most 20,000 characters. These bounds +apply before trimming, filtering, normalization, and template rendering. + +Malformed JSON and field values return short, content-safe errors. They name +only canonical fields where useful and never echo provider values or unknown +field names. The Promptkit adapter also drops an oversized provider result +before copying it into execution or debug state; direct executor implementations +receive the same enforcement in this package. + ## Render contexts The catalog's report-specific builders receive briefing metadata, a rich module diff --git a/docs/templates.md b/docs/templates.md index c11e31b..05391d9 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -127,13 +127,15 @@ It is not a source for deterministic weather facts. | Field | Hourly type | Daily, Today, and Tomorrow type | Notes | | --- | --- | --- | --- | -| `.GeneratedText.Summary` | `string` | `string` | Required. | -| `.GeneratedText.ForecastDiscussion` | `string` | `[]string` | Required; range over the day-style paragraph slice. | -| `.GeneratedText.PrecipitationTiming` | `string` | `string` | Required field; an empty string represents no supported prose. The precipitation partial uses nonempty prose only when deterministic windows exist. | +| `.GeneratedText.Summary` | `string` | `string` | Required; at most 4,000 characters. | +| `.GeneratedText.ForecastDiscussion` | `string` | `[]string` | Required; Hourly permits 12,000 characters. Day-style values permit up to 12 paragraphs of 4,000 characters each. | +| `.GeneratedText.PrecipitationTiming` | `string` | `string` | Required field, at most 4,000 characters; an empty string represents no supported prose. The precipitation partial uses nonempty prose only when deterministic windows exist. | The JSON schema rejects unknown properties and defines the required fields, but the schema body and validation behavior are documented in [Generated Text -internals](internal/generatedtext.md). +internals](internal/generatedtext.md). All validated generated prose together +is limited to 20,000 characters, so template edits can rely on a bounded prose +surface. ### Deterministic Module Values diff --git a/internal/adapters/promptkit/adapter.go b/internal/adapters/promptkit/adapter.go index 0b1ff99..6503989 100644 --- a/internal/adapters/promptkit/adapter.go +++ b/internal/adapters/promptkit/adapter.go @@ -9,6 +9,7 @@ import ( "time" promptkit "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptassets" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) @@ -193,6 +194,17 @@ func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.E value.Validation.SchemaPath, value.Validation.Errors, ) + rawOutput := []byte(nil) + if len(value.RawOutput) <= generatedtext.MaxGeneratedTextBytes { + rawOutput = []byte(value.RawOutput) + } else { + validation = promptexec.NewValidation( + promptexec.ValidationFailed, + string(value.Validation.Mode), + value.Validation.SchemaPath, + []string{"generated output exceeds the configured size limit"}, + ) + } execution := &promptexec.Execution{ RunID: value.RunID, PromptID: value.PromptID, @@ -215,11 +227,11 @@ func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.E EndedAt: value.EndTime, Duration: value.Duration, Validation: validation, - RawOutput: []byte(value.RawOutput), + RawOutput: rawOutput, } if captureDebug { execution.Debug = &promptexec.ExecutionDebug{ - RawOutput: append([]byte(nil), value.RawOutput...), + RawOutput: append([]byte(nil), rawOutput...), ValidationDiagnostics: append([]string(nil), validation.Diagnostics...), } } diff --git a/internal/adapters/promptkit/adapter_test.go b/internal/adapters/promptkit/adapter_test.go index 546972c..0b41515 100644 --- a/internal/adapters/promptkit/adapter_test.go +++ b/internal/adapters/promptkit/adapter_test.go @@ -12,6 +12,7 @@ import ( "time" promptkit "gitea.maximumdirect.net/eric/promptkit" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) @@ -364,6 +365,23 @@ func TestExecuteReturnsCompletedValidationRejection(t *testing.T) { } } +func TestExecuteDropsOversizedGeneratedOutput(t *testing.T) { + client := &fakeClient{response: &promptkit.GenerateResponse{Content: strings.Repeat("x", generatedtext.MaxGeneratedTextBytes+1)}} + adapter := newTestAdapter(t, client) + request := testExecuteRequest() + request.CaptureDebug = true + result, err := adapter.Execute(context.Background(), request, nil) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + if result == nil || result.Validation.Status != promptexec.ValidationFailed || len(result.RawOutput) != 0 || result.Debug == nil || len(result.Debug.RawOutput) != 0 { + t.Fatalf("execution = %#v", result) + } + if len(result.Validation.Diagnostics) != 1 || result.Validation.Diagnostics[0] != "generated output exceeds the configured size limit" { + t.Fatalf("diagnostics = %#v", result.Validation.Diagnostics) + } +} + func TestExecuteClassifiesOperationalFailures(t *testing.T) { tests := []struct { name string diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go index 9b66b0e..b46b996 100644 --- a/internal/app/profile_execution.go +++ b/internal/app/profile_execution.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) @@ -87,6 +88,9 @@ func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (p } outcome.ValidationStatus = execution.Validation.Status + if err := generatedtext.ValidateRawOutput(execution.RawOutput); err != nil { + return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err} + } if req.DebugWriter != nil && req.DebugWriter.Enabled() { if req.DebugRef == nil { return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))} diff --git a/internal/app/profile_execution_test.go b/internal/app/profile_execution_test.go index 5afaba6..68e10f9 100644 --- a/internal/app/profile_execution_test.go +++ b/internal/app/profile_execution_test.go @@ -5,9 +5,11 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) @@ -51,6 +53,23 @@ func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) { } } +func TestExecutePreparedProfileBoundsOversizedExecutorOutput(t *testing.T) { + prepared, inspection := preparedDailyProfile(t) + marker := "provider-controlled-marker" + executor := &generationExecutor{rawOutput: []byte(strings.Repeat("x", generatedtext.MaxGeneratedTextBytes+1) + marker)} + _, _, err := executePreparedProfile(context.Background(), profileExecutionRequest{ + Prepared: prepared, Prompt: inspection, + Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName}, + Executor: executor, + }) + if err == nil || !strings.Contains(err.Error(), "65536-byte limit") { + t.Fatalf("executePreparedProfile() error = %v, want bounded raw size error", err) + } + if len(err.Error()) > 160 || strings.Contains(err.Error(), marker) { + t.Fatalf("ordinary error leaked provider content: %q", err) + } +} + func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) { t.Helper() cfg := generationConfig() diff --git a/internal/generatedtext/daily_test.go b/internal/generatedtext/daily_test.go index 3bacb23..07162da 100644 --- a/internal/generatedtext/daily_test.go +++ b/internal/generatedtext/daily_test.go @@ -62,7 +62,7 @@ func TestValidateDailyRejectsInvalidInput(t *testing.T) { { name: "unknown field", in: `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"extra":"value"}`, - want: `unknown field "extra"`, + want: "unsupported field", }, { name: "missing summary", @@ -87,7 +87,7 @@ func TestValidateDailyRejectsInvalidInput(t *testing.T) { { name: "forecast discussion wrong type", in: `{"summary":"Showers are possible during the selected day.","forecast_discussion":"A front will keep rain chances in the forecast."}`, - want: "cannot unmarshal string", + want: "forecast discussion must be an array", }, { name: "multiple values", diff --git a/internal/generatedtext/day_style_test.go b/internal/generatedtext/day_style_test.go index 6ec073f..4e3f9a0 100644 --- a/internal/generatedtext/day_style_test.go +++ b/internal/generatedtext/day_style_test.go @@ -109,14 +109,14 @@ func TestValidateDayStyleGeneratedTextSharedBehavior(t *testing.T) { if err == nil { t.Fatal("validate() error = nil, want error") } - if !strings.Contains(err.Error(), `unknown field "extra"`) { - t.Fatalf("validate() error = %v, want unknown field error", err) + if !strings.Contains(err.Error(), "unsupported field") { + t.Fatalf("validate() error = %v, want unsupported field error", err) } }) t.Run("rejects retired confidence field", func(t *testing.T) { _, _, err := report.validate([]byte(`{"summary":"Shared summary.","forecast_discussion":["First paragraph."],"precipitation_timing":"","confidence":"Medium"}`)) - if err == nil || !strings.Contains(err.Error(), `unknown field "confidence"`) { + if err == nil || !strings.Contains(err.Error(), "unsupported field") { t.Fatalf("validate() error = %v, want retired confidence field rejection", err) } }) diff --git a/internal/generatedtext/hourly_test.go b/internal/generatedtext/hourly_test.go index 4260b16..985eb62 100644 --- a/internal/generatedtext/hourly_test.go +++ b/internal/generatedtext/hourly_test.go @@ -58,7 +58,7 @@ func TestValidateHourlyRejectsInvalidInput(t *testing.T) { { name: "unknown field", in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","extra":"value"}`, - want: `unknown field "extra"`, + want: "unsupported field", }, { name: "missing summary", @@ -83,17 +83,17 @@ func TestValidateHourlyRejectsInvalidInput(t *testing.T) { { name: "retired confidence field rejected", in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"","confidence":"Medium"}`, - want: `unknown field "confidence"`, + want: "unsupported field", }, { name: "old timing field rejected", in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","timing":"Late morning."}`, - want: `unknown field "timing"`, + want: "unsupported field", }, { name: "old impacts field rejected", in: `{"summary":"Storm chances increase.","forecast_discussion":"A front will keep the region unsettled.","impacts":"Brief downpours."}`, - want: `unknown field "impacts"`, + want: "unsupported field", }, { name: "multiple values", diff --git a/internal/generatedtext/json.go b/internal/generatedtext/json.go index 8e07e8c..81f1edc 100644 --- a/internal/generatedtext/json.go +++ b/internal/generatedtext/json.go @@ -16,18 +16,21 @@ var generatedTextFieldNames = map[string]struct{}{ func decodeGeneratedText[T any](data []byte, name string) (T, error) { var value T + if err := validateGeneratedTextSize(data, name); err != nil { + return value, err + } if err := validateGeneratedTextObject(data, name); err != nil { return value, err } decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(&value); err != nil { - return value, fmt.Errorf("decode %s generated text: %w", name, err) + return value, fmt.Errorf("decode %s generated text: invalid field value", name) } var extra any if err := decoder.Decode(&extra); err != nil { if err != io.EOF { - return value, fmt.Errorf("decode %s generated text: %w", name, err) + return value, fmt.Errorf("decode %s generated text: invalid JSON", name) } return value, nil } @@ -38,35 +41,39 @@ func validateGeneratedTextObject(data []byte, name string) error { decoder := json.NewDecoder(bytes.NewReader(data)) token, err := decoder.Token() if err != nil { - return fmt.Errorf("decode %s generated text: %w", name, err) + return fmt.Errorf("decode %s generated text: invalid JSON", name) } if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' { return fmt.Errorf("decode %s generated text: expected JSON object", name) } seen := make(map[string]struct{}, len(generatedTextFieldNames)) + totalCharacters := 0 for decoder.More() { token, err := decoder.Token() if err != nil { - return fmt.Errorf("decode %s generated text: %w", name, err) + return fmt.Errorf("decode %s generated text: invalid JSON", name) } field, ok := token.(string) if !ok { return fmt.Errorf("decode %s generated text: expected object field name", name) } if _, ok := generatedTextFieldNames[field]; !ok { - return fmt.Errorf("decode %s generated text: json: unknown field %q", name, field) + return fmt.Errorf("decode %s generated text: unsupported field", name) } if _, ok := seen[field]; ok { - return fmt.Errorf("decode %s generated text: duplicate field %q", name, field) + return fmt.Errorf("decode %s generated text: duplicate %s field", name, generatedTextFieldLabel(field)) } seen[field] = struct{}{} var value json.RawMessage if err := decoder.Decode(&value); err != nil { - return fmt.Errorf("decode %s generated text: %w", name, err) + return fmt.Errorf("decode %s generated text: invalid JSON", name) + } + if err := validateGeneratedTextField(name, field, value, &totalCharacters); err != nil { + return err } } if _, err := decoder.Token(); err != nil { - return fmt.Errorf("decode %s generated text: %w", name, err) + return fmt.Errorf("decode %s generated text: invalid JSON", name) } return nil } @@ -74,15 +81,12 @@ func validateGeneratedTextObject(data []byte, name string) error { func requireGeneratedTextStringField(data []byte, name, field string) error { var fields map[string]json.RawMessage if err := json.Unmarshal(data, &fields); err != nil { - return fmt.Errorf("decode %s generated text: %w", name, err) + return fmt.Errorf("decode %s generated text: invalid JSON", name) } - raw, ok := fields[field] + _, ok := fields[field] if !ok { return fmt.Errorf("%s generated text %s is required", name, strings.ReplaceAll(field, "_", " ")) } - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - return fmt.Errorf("%s generated text %s must be a string", name, strings.ReplaceAll(field, "_", " ")) - } return nil } diff --git a/internal/generatedtext/json_test.go b/internal/generatedtext/json_test.go index 6f39618..1255cfa 100644 --- a/internal/generatedtext/json_test.go +++ b/internal/generatedtext/json_test.go @@ -87,6 +87,56 @@ func TestTypedValidatorsMatchEmbeddedSchemaObjectShape(t *testing.T) { } } +func TestTypedValidatorsMatchEmbeddedSchemaContentLimits(t *testing.T) { + validators := []struct { + name string + valid string + invalid []string + validate func([]byte) error + }{ + { + name: "daily", + valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)), + invalid: []string{fmt.Sprintf(`{"summary":%q,"forecast_discussion":["discussion"],"precipitation_timing":""}`, strings.Repeat("s", 4_001)), fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"discussion",`, 13), ",")), `{"summary":"summary","forecast_discussion":[null],"precipitation_timing":""}`}, + validate: validateDailyJSON, + }, + { + name: "today", + valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)), + invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("d", 4_001))}, + validate: validateTodayJSON, + }, + { + name: "tomorrow", + valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 4_000)), + invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":["discussion"],"precipitation_timing":%q}`, strings.Repeat("t", 4_001))}, + validate: validateTomorrowJSON, + }, + { + name: "hourly", + valid: fmt.Sprintf(`{"summary":%q,"forecast_discussion":%q,"precipitation_timing":""}`, strings.Repeat("s", 4_000), strings.Repeat("d", 12_000)), + invalid: []string{fmt.Sprintf(`{"summary":"summary","forecast_discussion":%q,"precipitation_timing":""}`, strings.Repeat("d", 12_001))}, + validate: validateHourlyJSON, + }, + } + for _, validator := range validators { + t.Run(validator.name, func(t *testing.T) { + schema := generatedTextSchema(t, validator.name) + for _, input := range append([]string{validator.valid}, validator.invalid...) { + instance, err := jsonschema.UnmarshalJSON(strings.NewReader(input)) + if err != nil { + t.Fatalf("parse schema instance: %v", err) + } + schemaValid := schema.Validate(instance) == nil + typedValid := validator.validate([]byte(input)) == nil + if schemaValid != typedValid { + t.Fatalf("schema valid = %t, typed valid = %t", schemaValid, typedValid) + } + } + }) + } +} + func generatedTextSchema(t *testing.T, id string) *jsonschema.Schema { t.Helper() data, err := promptassets.Schema(id) diff --git a/internal/generatedtext/limits.go b/internal/generatedtext/limits.go new file mode 100644 index 0000000..439eadf --- /dev/null +++ b/internal/generatedtext/limits.go @@ -0,0 +1,120 @@ +package generatedtext + +import ( + "bytes" + "encoding/json" + "fmt" + "unicode/utf8" +) + +const ( + // MaxGeneratedTextBytes is the largest provider response accepted for one + // generated-text report before JSON decoding begins. + MaxGeneratedTextBytes = 64 * 1024 + + maxGeneratedTextCharacters = 20_000 + maxGeneratedTextStringCharacters = 4_000 + maxHourlyDiscussionCharacters = 12_000 + maxDayStyleDiscussionParagraphs = 12 + maxDayStyleDiscussionParagraphChars = 4_000 +) + +// ValidateRawOutput rejects generated JSON before it is decoded or retained. +func ValidateRawOutput(data []byte) error { + if len(data) > MaxGeneratedTextBytes { + return fmt.Errorf("generated text exceeds the %d-byte limit", MaxGeneratedTextBytes) + } + return nil +} + +func validateGeneratedTextSize(data []byte, name string) error { + if err := ValidateRawOutput(data); err != nil { + return fmt.Errorf("%s %w", name, err) + } + return nil +} + +func validateGeneratedTextField(name, field string, raw json.RawMessage, total *int) error { + switch field { + case "summary", "precipitation_timing": + return validateGeneratedTextString(name, field, raw, maxGeneratedTextStringCharacters, total) + case "forecast_discussion": + if name == "hourly" { + return validateGeneratedTextString(name, field, raw, maxHourlyDiscussionCharacters, total) + } + return validateDayStyleDiscussion(name, raw, total) + default: + return fmt.Errorf("decode %s generated text: unsupported field", name) + } +} + +func validateGeneratedTextString(name, field string, raw json.RawMessage, limit int, total *int) error { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("%s generated text %s must be a string", name, generatedTextFieldLabel(field)) + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return fmt.Errorf("%s generated text %s must be a string", name, generatedTextFieldLabel(field)) + } + characters := utf8.RuneCountInString(value) + if characters > limit { + return fmt.Errorf("%s generated text %s exceeds the %d-character limit", name, generatedTextFieldLabel(field), limit) + } + return addGeneratedTextCharacters(name, characters, total) +} + +func validateDayStyleDiscussion(name string, raw json.RawMessage, total *int) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil { + return fmt.Errorf("%s generated text forecast discussion must be an array", name) + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '[' { + return fmt.Errorf("%s generated text forecast discussion must be an array", name) + } + + paragraphs := 0 + for decoder.More() { + paragraphs++ + if paragraphs > maxDayStyleDiscussionParagraphs { + return fmt.Errorf("%s generated text forecast discussion exceeds the %d-paragraph limit", name, maxDayStyleDiscussionParagraphs) + } + var rawParagraph json.RawMessage + if err := decoder.Decode(&rawParagraph); err != nil || bytes.Equal(bytes.TrimSpace(rawParagraph), []byte("null")) { + return fmt.Errorf("%s generated text forecast discussion paragraphs must be strings", name) + } + var paragraph string + if err := json.Unmarshal(rawParagraph, ¶graph); err != nil { + return fmt.Errorf("%s generated text forecast discussion paragraphs must be strings", name) + } + characters := utf8.RuneCountInString(paragraph) + if characters > maxDayStyleDiscussionParagraphChars { + return fmt.Errorf("%s generated text forecast discussion paragraphs exceed the %d-character limit", name, maxDayStyleDiscussionParagraphChars) + } + if err := addGeneratedTextCharacters(name, characters, total); err != nil { + return err + } + } + if _, err := decoder.Token(); err != nil { + return fmt.Errorf("%s generated text forecast discussion must be an array", name) + } + return nil +} + +func addGeneratedTextCharacters(name string, characters int, total *int) error { + *total += characters + if *total > maxGeneratedTextCharacters { + return fmt.Errorf("%s generated text exceeds the %d-character limit", name, maxGeneratedTextCharacters) + } + return nil +} + +func generatedTextFieldLabel(field string) string { + if field == "forecast_discussion" { + return "forecast discussion" + } + if field == "precipitation_timing" { + return "precipitation timing" + } + return field +} diff --git a/internal/generatedtext/limits_test.go b/internal/generatedtext/limits_test.go new file mode 100644 index 0000000..a7a828b --- /dev/null +++ b/internal/generatedtext/limits_test.go @@ -0,0 +1,112 @@ +package generatedtext + +import ( + "fmt" + "strings" + "testing" +) + +func TestValidateHourlyGeneratedTextEnforcesContentLimits(t *testing.T) { + input := func(summary, discussion, timing string) []byte { + return []byte(fmt.Sprintf(`{"summary":%q,"forecast_discussion":%q,"precipitation_timing":%q}`, summary, discussion, timing)) + } + for _, test := range []struct { + name string + input []byte + want string + }{ + {name: "summary at limit", input: input(strings.Repeat("s", maxGeneratedTextStringCharacters), "discussion", "")}, + {name: "summary over limit", input: input(strings.Repeat("s", maxGeneratedTextStringCharacters+1), "discussion", ""), want: "summary exceeds"}, + {name: "discussion at limit", input: input("summary", strings.Repeat("d", maxHourlyDiscussionCharacters), "")}, + {name: "discussion over limit", input: input("summary", strings.Repeat("d", maxHourlyDiscussionCharacters+1), ""), want: "forecast discussion exceeds"}, + {name: "total at limit", input: input(strings.Repeat("s", 4_000), strings.Repeat("d", 12_000), strings.Repeat("t", 4_000))}, + } { + t.Run(test.name, func(t *testing.T) { + err := validateHourlyJSON(test.input) + if test.want == "" && err != nil { + t.Fatalf("validate() error = %v", err) + } + if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) { + t.Fatalf("validate() error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateDayStyleGeneratedTextEnforcesDiscussionLimits(t *testing.T) { + for _, test := range []struct { + name string + input string + want string + }{ + { + name: "paragraph count at limit", + input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"p",`, maxDayStyleDiscussionParagraphs), ",")), + }, + { + name: "paragraph count over limit", + input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%s],"precipitation_timing":""}`, strings.TrimSuffix(strings.Repeat(`"p",`, maxDayStyleDiscussionParagraphs+1), ",")), + want: "12-paragraph limit", + }, + { + name: "paragraph at limit", + input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("p", maxDayStyleDiscussionParagraphChars)), + }, + { + name: "paragraph over limit", + input: fmt.Sprintf(`{"summary":"summary","forecast_discussion":[%q],"precipitation_timing":""}`, strings.Repeat("p", maxDayStyleDiscussionParagraphChars+1)), + want: "paragraphs exceed the 4000-character limit", + }, + { + name: "total at limit", + input: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q,%q,%q,%q],"precipitation_timing":""}`, + strings.Repeat("s", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000)), + }, + { + name: "total over limit", + input: fmt.Sprintf(`{"summary":%q,"forecast_discussion":[%q,%q,%q,%q,%q],"precipitation_timing":""}`, + strings.Repeat("s", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000), strings.Repeat("p", 4_000)), + want: "exceeds the 20000-character limit", + }, + } { + t.Run(test.name, func(t *testing.T) { + err := validateDailyJSON([]byte(test.input)) + if test.want == "" && err != nil { + t.Fatalf("validate() error = %v", err) + } + if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) { + t.Fatalf("validate() error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateGeneratedTextRejectsOversizedRawOutputWithoutLeakingContent(t *testing.T) { + valid := []byte(`{"summary":"summary","forecast_discussion":"discussion","precipitation_timing":""}`) + atLimit := append([]byte(strings.Repeat(" ", MaxGeneratedTextBytes-len(valid))), valid...) + if err := validateHourlyJSON(atLimit); err != nil { + t.Fatalf("validate() at raw limit error = %v", err) + } + + marker := "provider-controlled-marker" + unknownPrefix := `{"summary":"summary","forecast_discussion":"discussion","precipitation_timing":"","` + unknownSuffix := `":"value"}` + unknownKey := marker + strings.Repeat("x", MaxGeneratedTextBytes-len(unknownPrefix)-len(unknownSuffix)-len(marker)) + unknownField := []byte(unknownPrefix + unknownKey + unknownSuffix) + err := validateHourlyJSON(unknownField) + if err == nil || !strings.Contains(err.Error(), "unsupported field") { + t.Fatalf("validate() error = %v, want unsupported field error", err) + } + if len(err.Error()) > 128 || strings.Contains(err.Error(), marker) { + t.Fatalf("validate() leaked unknown field content: %q", err) + } + + input := append(atLimit, []byte(marker)...) + err = validateHourlyJSON(input) + if err == nil || !strings.Contains(err.Error(), "65536-byte limit") { + t.Fatalf("validate() error = %v, want raw size error", err) + } + if len(err.Error()) > 128 || strings.Contains(err.Error(), marker) { + t.Fatalf("validate() leaked oversized provider content: %q", err) + } +} diff --git a/internal/generatedtext/today_test.go b/internal/generatedtext/today_test.go index cff56cb..03a8e42 100644 --- a/internal/generatedtext/today_test.go +++ b/internal/generatedtext/today_test.go @@ -62,7 +62,7 @@ func TestValidateTodayRejectsInvalidInput(t *testing.T) { { name: "unknown field", in: `{"summary":"Showers are likely today.","forecast_discussion":["A front will keep rain chances elevated."],"extra":"value"}`, - want: `unknown field "extra"`, + want: "unsupported field", }, { name: "missing summary", @@ -87,7 +87,7 @@ func TestValidateTodayRejectsInvalidInput(t *testing.T) { { name: "forecast discussion wrong type", in: `{"summary":"Showers are likely today.","forecast_discussion":"A front will keep rain chances elevated."}`, - want: "cannot unmarshal string", + want: "forecast discussion must be an array", }, { name: "multiple values", diff --git a/internal/generatedtext/tomorrow_test.go b/internal/generatedtext/tomorrow_test.go index f8aed4c..ca5b490 100644 --- a/internal/generatedtext/tomorrow_test.go +++ b/internal/generatedtext/tomorrow_test.go @@ -62,7 +62,7 @@ func TestValidateTomorrowRejectsInvalidInput(t *testing.T) { { name: "unknown field", in: `{"summary":"Storms become more likely tomorrow.","forecast_discussion":["A front will keep showers in the forecast."],"extra":"value"}`, - want: `unknown field "extra"`, + want: "unsupported field", }, { name: "missing summary", @@ -87,7 +87,7 @@ func TestValidateTomorrowRejectsInvalidInput(t *testing.T) { { name: "forecast discussion wrong type", in: `{"summary":"Storms become more likely tomorrow.","forecast_discussion":"A front will keep showers in the forecast."}`, - want: "cannot unmarshal string", + want: "forecast discussion must be an array", }, { name: "multiple values", diff --git a/internal/promptassets/assets/schemas/daily.generated_text.schema.json b/internal/promptassets/assets/schemas/daily.generated_text.schema.json index 21b1de0..ccee4e1 100644 --- a/internal/promptassets/assets/schemas/daily.generated_text.schema.json +++ b/internal/promptassets/assets/schemas/daily.generated_text.schema.json @@ -6,8 +6,8 @@ "additionalProperties": false, "required": ["summary", "forecast_discussion", "precipitation_timing"], "properties": { - "summary": {"type": "string"}, - "forecast_discussion": {"type": "array", "items": {"type": "string"}, "minItems": 1}, - "precipitation_timing": {"type": "string"} + "summary": {"type": "string", "maxLength": 4000}, + "forecast_discussion": {"type": "array", "items": {"type": "string", "maxLength": 4000}, "minItems": 1, "maxItems": 12}, + "precipitation_timing": {"type": "string", "maxLength": 4000} } } diff --git a/internal/promptassets/assets/schemas/hourly.generated_text.schema.json b/internal/promptassets/assets/schemas/hourly.generated_text.schema.json index 45c362f..d64edea 100644 --- a/internal/promptassets/assets/schemas/hourly.generated_text.schema.json +++ b/internal/promptassets/assets/schemas/hourly.generated_text.schema.json @@ -6,8 +6,8 @@ "additionalProperties": false, "required": ["summary", "forecast_discussion", "precipitation_timing"], "properties": { - "summary": {"type": "string"}, - "forecast_discussion": {"type": "string"}, - "precipitation_timing": {"type": "string"} + "summary": {"type": "string", "maxLength": 4000}, + "forecast_discussion": {"type": "string", "maxLength": 12000}, + "precipitation_timing": {"type": "string", "maxLength": 4000} } } diff --git a/internal/promptassets/assets/schemas/today.generated_text.schema.json b/internal/promptassets/assets/schemas/today.generated_text.schema.json index 1fad04b..9ba9532 100644 --- a/internal/promptassets/assets/schemas/today.generated_text.schema.json +++ b/internal/promptassets/assets/schemas/today.generated_text.schema.json @@ -6,8 +6,8 @@ "additionalProperties": false, "required": ["summary", "forecast_discussion", "precipitation_timing"], "properties": { - "summary": {"type": "string"}, - "forecast_discussion": {"type": "array", "items": {"type": "string"}, "minItems": 1}, - "precipitation_timing": {"type": "string"} + "summary": {"type": "string", "maxLength": 4000}, + "forecast_discussion": {"type": "array", "items": {"type": "string", "maxLength": 4000}, "minItems": 1, "maxItems": 12}, + "precipitation_timing": {"type": "string", "maxLength": 4000} } } diff --git a/internal/promptassets/assets/schemas/tomorrow.generated_text.schema.json b/internal/promptassets/assets/schemas/tomorrow.generated_text.schema.json index bff0368..3165d72 100644 --- a/internal/promptassets/assets/schemas/tomorrow.generated_text.schema.json +++ b/internal/promptassets/assets/schemas/tomorrow.generated_text.schema.json @@ -6,8 +6,8 @@ "additionalProperties": false, "required": ["summary", "forecast_discussion", "precipitation_timing"], "properties": { - "summary": {"type": "string"}, - "forecast_discussion": {"type": "array", "items": {"type": "string"}, "minItems": 1}, - "precipitation_timing": {"type": "string"} + "summary": {"type": "string", "maxLength": 4000}, + "forecast_discussion": {"type": "array", "items": {"type": "string", "maxLength": 4000}, "minItems": 1, "maxItems": 12}, + "precipitation_timing": {"type": "string", "maxLength": 4000} } }