Refactor the yaml data package to group data sources by category

docs/internal/prompt-input.md

@@ -24,7 +24,8 @@ Inputs:
 Outputs:
 
 - `promptinput.Package` with schema version, RunID, report metadata, named
-  module stanzas, Recent Changes, and source warnings
+  module stanzas grouped for prompt presentation, Recent Changes, and source
+  warnings
 - YAML bytes from `promptinput.MarshalYAML`
 - YAML file written atomically by `promptinput.Save`
 
@@ -38,15 +39,37 @@ report:
   prompt_id: <prompt_id>
 briefing:
   metadata: {}
-  current_conditions: {}
+  applicable_risk_products:
+    alert_digest: {}
+  derived_summaries:
+    derived_daily_summary: {}
+    derived_daypart_summaries: {}
+    precip_timing: {}
+    outdoor_windows: {}
+  narrative_products:
     narrative_forecast: {}
+    area_forecast_discussion: {}
+    weather_story: {}
+  raw_data:
+    current_conditions: {}
     hourly_forecast: {}
 recent_changes:
   items: []
 ```
 
-The `briefing` mapping contains named module stanzas. Stanza order follows the
+The `briefing` mapping keeps `metadata` directly under `briefing` and groups
-module snapshot output order.
+weather module stanzas under prompt-facing categories. This grouping is a YAML
+presentation concern only: module snapshots remain flat, and loaded
+`promptinput.Package` values expose flat stanza names in `Briefing.Values`.
+Within each category, stanza order follows the module snapshot output order.
+
+Current categories are:
+
+- `applicable_risk_products`: location-applicable alerts, warnings, outlooks,
+  discussions, and similar risk products.
+- `derived_summaries`: deterministic summaries and calculated report facts.
+- `narrative_products`: official narrative text products and forecast stories.
+- `raw_data`: minimally transformed underlying weather data.
 
 ## Boundaries
 
@@ -92,6 +115,7 @@ Inspect:
 ## Invariants
 
 - Scriptorium receives structured YAML through `--input data_package=<path>`.
-- Module stanza order is deterministic for generated snapshots.
+- Module stanza order is deterministic within each prompt-facing category.
+- Every non-metadata module stanza has exactly one prompt-input category.
 - Recent Changes are provided by `internal/changes`; this package does not
   infer changes from rendered report text.

internal/app/app_test.go

@@ -160,6 +160,10 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) {
 	}
 	if !strings.Contains(string(data), "schema_version: weatherreporter.data_package.v2") ||
 		!strings.Contains(string(data), "recent_changes:") ||
+		!strings.Contains(string(data), "applicable_risk_products:") ||
+		!strings.Contains(string(data), "derived_summaries:") ||
+		!strings.Contains(string(data), "narrative_products:") ||
+		!strings.Contains(string(data), "raw_data:") ||
 		!strings.Contains(string(data), "current_conditions:") ||
 		!strings.Contains(string(data), "narrative_forecast:") ||
 		!strings.Contains(string(data), "hourly_forecast:") ||
@@ -169,13 +173,19 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) {
 	if strings.Contains(string(data), "source_warnings:") {
 		t.Fatalf("data package has source warnings, want none for complete fetched sources:\n%s", string(data))
 	}
-	currentIndex := strings.Index(string(data), "  current_conditions:")
+	riskIndex := strings.Index(string(data), "  applicable_risk_products:")
-	narrativeIndex := strings.Index(string(data), "  narrative_forecast:")
+	derivedIndex := strings.Index(string(data), "  derived_summaries:")
+	narrativeIndex := strings.Index(string(data), "  narrative_products:")
+	rawIndex := strings.Index(string(data), "  raw_data:")
+	alertIndex := strings.Index(string(data), "    alert_digest:")
 	summaryIndex := strings.Index(string(data), "    derived_daily_summary:")
-	outdoorIndex := strings.Index(string(data), "  outdoor_windows:")
+	storyIndex := strings.Index(string(data), "    weather_story:")
+	currentIndex := strings.Index(string(data), "    current_conditions:")
 	hourlyIndex := strings.Index(string(data), "    hourly_forecast:")
-	if currentIndex < 0 || narrativeIndex < 0 || summaryIndex < 0 || outdoorIndex < 0 || hourlyIndex < 0 || !(currentIndex < narrativeIndex && narrativeIndex < summaryIndex && outdoorIndex < hourlyIndex) {
+	if riskIndex < 0 || derivedIndex < 0 || narrativeIndex < 0 || rawIndex < 0 || alertIndex < 0 || summaryIndex < 0 || storyIndex < 0 || currentIndex < 0 || hourlyIndex < 0 ||
-		t.Fatalf("data package stanza order is wrong, want current_conditions then narrative_forecast then derived_daily_summary and final hourly_forecast:\n%s", string(data))
+		!(riskIndex < derivedIndex && derivedIndex < narrativeIndex && narrativeIndex < rawIndex) ||
+		!(riskIndex < alertIndex && derivedIndex < summaryIndex && narrativeIndex < storyIndex && rawIndex < currentIndex && currentIndex < hourlyIndex) {
+		t.Fatalf("data package grouping is wrong, want categorized prompt stanzas:\n%s", string(data))
 	}
 	savedDataPackage, err := promptinput.LoadYAML(data)
 	if err != nil {

internal/promptinput/package.go

@@ -18,6 +18,35 @@ import (
 
 const SchemaVersion = "weatherreporter.data_package.v2"
 
+const (
+	metadataStanza                 = "metadata"
+	categoryApplicableRiskProducts = "applicable_risk_products"
+	categoryDerivedSummaries       = "derived_summaries"
+	categoryNarrativeProducts      = "narrative_products"
+	categoryRawData                = "raw_data"
+)
+
+var briefingCategoryOrder = []string{
+	categoryApplicableRiskProducts,
+	categoryDerivedSummaries,
+	categoryNarrativeProducts,
+	categoryRawData,
+}
+
+var briefingStanzaCategories = map[string]string{
+	string(module.AlertDigest):             categoryApplicableRiskProducts,
+	string(module.DerivedDailySummary):     categoryDerivedSummaries,
+	string(module.DerivedDaypartSummaries): categoryDerivedSummaries,
+	string(module.PrecipTiming):            categoryDerivedSummaries,
+	string(module.OutdoorWindows):          categoryDerivedSummaries,
+	string(module.TomorrowPlanning):        categoryDerivedSummaries,
+	string(module.NarrativeForecast):       categoryNarrativeProducts,
+	string(module.AreaForecastDiscussion):  categoryNarrativeProducts,
+	string(module.WeatherStory):            categoryNarrativeProducts,
+	string(module.CurrentConditions):       categoryRawData,
+	string(module.HourlyForecast):          categoryRawData,
+}
+
 type BuildRequest struct {
 	Metadata      Metadata
 	Modules       module.Snapshot
@@ -144,13 +173,24 @@ func Validate(pkg Package) error {
 	if len(pkg.Briefing.Order) == 0 {
 		return fmt.Errorf("briefing stanzas are required")
 	}
+	seen := map[string]struct{}{}
 	for _, name := range pkg.Briefing.Order {
 		if name == "" {
 			return fmt.Errorf("briefing stanza name is required")
 		}
+		if _, ok := seen[name]; ok {
+			return fmt.Errorf("duplicate briefing stanza %q", name)
+		}
+		seen[name] = struct{}{}
 		if _, ok := pkg.Briefing.Values[name]; !ok {
 			return fmt.Errorf("briefing stanza %q is missing", name)
 		}
+		if name == metadataStanza {
+			continue
+		}
+		if _, ok := briefingStanzaCategories[name]; !ok {
+			return fmt.Errorf("briefing stanza %q has no prompt-input category", name)
+		}
 	}
 	return nil
 }
@@ -191,17 +231,40 @@ func LoadYAML(data []byte) (Package, error) {
 
 func (b BriefingStanzas) MarshalYAML() (any, error) {
 	node := &yaml.Node{Kind: yaml.MappingNode}
+	categoryNames := map[string][]string{}
 	for _, name := range b.Order {
 		value, ok := b.Values[name]
 		if !ok {
 			continue
 		}
-		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: name}
+		if name == metadataStanza {
-		valueNode, err := yamlNode(value)
+			if err := appendYAMLMappingValue(node, name, value); err != nil {
-		if err != nil {
+				return nil, err
-			return nil, fmt.Errorf("marshal briefing stanza %q: %w", name, err)
 			}
-		node.Content = append(node.Content, keyNode, valueNode)
+			continue
+		}
+		category, ok := briefingStanzaCategories[name]
+		if !ok {
+			return nil, fmt.Errorf("briefing stanza %q has no prompt-input category", name)
+		}
+		categoryNames[category] = append(categoryNames[category], name)
+	}
+	for _, category := range briefingCategoryOrder {
+		names := categoryNames[category]
+		if len(names) == 0 {
+			continue
+		}
+		categoryNode := &yaml.Node{Kind: yaml.MappingNode}
+		for _, name := range names {
+			value := b.Values[name]
+			if err := appendYAMLMappingValue(categoryNode, name, value); err != nil {
+				return nil, err
+			}
+		}
+		node.Content = append(node.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: category},
+			categoryNode,
+		)
 	}
 	return node, nil
 }
@@ -212,14 +275,46 @@ func (b *BriefingStanzas) UnmarshalYAML(value *yaml.Node) error {
 	}
 	values := map[string]any{}
 	order := make([]string, 0, len(value.Content)/2)
+	seen := map[string]struct{}{}
+	seenCategories := map[string]struct{}{}
+	categoryOrder := map[string][]string{}
 	for i := 0; i < len(value.Content); i += 2 {
 		name := value.Content[i].Value
-		var stanza any
+		if name == metadataStanza {
-		if err := value.Content[i+1].Decode(&stanza); err != nil {
+			if err := decodeBriefingStanza(value.Content[i+1], name, values, &order, seen); err != nil {
 				return err
 			}
-		order = append(order, name)
+			continue
-		values[name] = stanza
+		}
+		if !knownBriefingCategory(name) {
+			return fmt.Errorf("unknown briefing category %q", name)
+		}
+		if _, ok := seenCategories[name]; ok {
+			return fmt.Errorf("duplicate briefing category %q", name)
+		}
+		seenCategories[name] = struct{}{}
+		categoryNode := value.Content[i+1]
+		if categoryNode.Kind != yaml.MappingNode {
+			return fmt.Errorf("briefing category %q must be a mapping", name)
+		}
+		var names []string
+		for j := 0; j < len(categoryNode.Content); j += 2 {
+			stanzaName := categoryNode.Content[j].Value
+			category, ok := briefingStanzaCategories[stanzaName]
+			if !ok {
+				return fmt.Errorf("briefing stanza %q has no prompt-input category", stanzaName)
+			}
+			if category != name {
+				return fmt.Errorf("briefing stanza %q belongs under category %q, not %q", stanzaName, category, name)
+			}
+			if err := decodeBriefingStanza(categoryNode.Content[j+1], stanzaName, values, &names, seen); err != nil {
+				return err
+			}
+		}
+		categoryOrder[name] = names
+	}
+	for _, category := range briefingCategoryOrder {
+		order = append(order, categoryOrder[category]...)
 	}
 	b.Order = order
 	b.Values = values
@@ -250,6 +345,39 @@ func (b *BriefingStanzas) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+func appendYAMLMappingValue(node *yaml.Node, name string, value any) error {
+	keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: name}
+	valueNode, err := yamlNode(value)
+	if err != nil {
+		return fmt.Errorf("marshal briefing stanza %q: %w", name, err)
+	}
+	node.Content = append(node.Content, keyNode, valueNode)
+	return nil
+}
+
+func decodeBriefingStanza(node *yaml.Node, name string, values map[string]any, order *[]string, seen map[string]struct{}) error {
+	if _, ok := seen[name]; ok {
+		return fmt.Errorf("duplicate briefing stanza %q", name)
+	}
+	var stanza any
+	if err := node.Decode(&stanza); err != nil {
+		return err
+	}
+	seen[name] = struct{}{}
+	*order = append(*order, name)
+	values[name] = stanza
+	return nil
+}
+
+func knownBriefingCategory(name string) bool {
+	for _, category := range briefingCategoryOrder {
+		if name == category {
+			return true
+		}
+	}
+	return false
+}
+
 func yamlNode(value any) (*yaml.Node, error) {
 	data, err := json.Marshal(value)
 	if err != nil {

internal/promptinput/package_test.go

@@ -106,7 +106,7 @@ func TestBuildUsesNamedSnapshotStanzas(t *testing.T) {
 	req.Metadata.PromptID = "weather.three_day_outlook"
 	req.Modules = snapshotWithOutputs(t,
 		module.Output{ID: module.Metadata, StanzaName: "metadata", Value: map[string]string{"run_id": req.Metadata.RunID}},
-		module.Output{ID: module.DerivedDaypartSummaries, StanzaName: "three_day", Value: map[string]any{"days": []string{"2026-05-29"}}},
+		module.Output{ID: module.DerivedDaypartSummaries, StanzaName: "derived_daypart_summaries", Value: map[string]any{"days": []string{"2026-05-29"}}},
 	)
 
 	pkg, err := Build(req)
@@ -117,12 +117,12 @@ func TestBuildUsesNamedSnapshotStanzas(t *testing.T) {
 	if pkg.Report.ID != report.ThreeDay {
 		t.Fatalf("Report.ID = %q, want three_day", pkg.Report.ID)
 	}
-	if _, ok := pkg.Briefing.Values["three_day"]; !ok {
+	if _, ok := pkg.Briefing.Values["derived_daypart_summaries"]; !ok {
-		t.Fatal("Briefing.Values[three_day] missing")
+		t.Fatal("Briefing.Values[derived_daypart_summaries] missing")
 	}
 }
 
-func TestMarshalYAMLIsDeterministicAndUsesNamedStanzas(t *testing.T) {
+func TestMarshalYAMLIsDeterministicAndGroupsNamedStanzas(t *testing.T) {
 	pkg, err := Build(validBuildRequest(t))
 	if err != nil {
 		t.Fatalf("Build() error = %v", err)
@@ -141,9 +141,26 @@ func TestMarshalYAMLIsDeterministicAndUsesNamedStanzas(t *testing.T) {
 	}
 	if !strings.Contains(string(first), "schema_version: weatherreporter.data_package.v2") ||
 		!strings.Contains(string(first), "briefing:\n") ||
+		!strings.Contains(string(first), "  applicable_risk_products:\n") ||
+		!strings.Contains(string(first), "  derived_summaries:\n") ||
+		!strings.Contains(string(first), "  narrative_products:\n") ||
+		!strings.Contains(string(first), "  raw_data:\n") ||
 		!strings.Contains(string(first), "    current_conditions:\n") ||
 		!strings.Contains(string(first), "      condition_text: Partly cloudy") {
-		t.Fatalf("YAML output missing expected named stanzas:\n%s", string(first))
+		t.Fatalf("YAML output missing expected grouped stanzas:\n%s", string(first))
+	}
+	for _, pair := range []struct {
+		before string
+		after  string
+	}{
+		{before: "  metadata:\n", after: "  applicable_risk_products:\n"},
+		{before: "  applicable_risk_products:\n", after: "  derived_summaries:\n"},
+		{before: "  derived_summaries:\n", after: "  narrative_products:\n"},
+		{before: "  narrative_products:\n", after: "  raw_data:\n"},
+	} {
+		if strings.Index(string(first), pair.before) < 0 || strings.Index(string(first), pair.after) < 0 || strings.Index(string(first), pair.before) > strings.Index(string(first), pair.after) {
+			t.Fatalf("YAML category order is wrong, want %q before %q:\n%s", pair.before, pair.after, string(first))
+		}
 	}
 }
 
@@ -165,8 +182,51 @@ func TestLoadYAMLRoundTrip(t *testing.T) {
 	if loaded.SchemaVersion != SchemaVersion || loaded.RunID != pkg.RunID {
 		t.Fatalf("loaded package = %#v, want schema and run id", loaded)
 	}
-	if loaded.Briefing.Order[1] != "current_conditions" {
+	wantOrder := []string{"metadata", "alert_digest", "derived_daily_summary", "narrative_forecast", "current_conditions"}
-		t.Fatalf("loaded package order = %#v, want current_conditions second", loaded.Briefing.Order)
+	if strings.Join(loaded.Briefing.Order, ",") != strings.Join(wantOrder, ",") {
+		t.Fatalf("loaded package order = %#v, want grouped category order %#v", loaded.Briefing.Order, wantOrder)
+	}
+	if got := loaded.Briefing.Values["current_conditions"].(map[string]any)["condition_text"]; got != "Partly cloudy" {
+		t.Fatalf("loaded current_conditions.condition_text = %#v, want Partly cloudy", got)
+	}
+}
+
+func TestMarshalYAMLRejectsUncategorizedStanza(t *testing.T) {
+	req := validBuildRequest(t)
+	req.Modules = snapshotWithOutputs(t, module.Output{ID: module.ID("custom"), StanzaName: "custom", Value: map[string]string{"value": "x"}})
+
+	_, err := Build(req)
+	if err == nil || !strings.Contains(err.Error(), `briefing stanza "custom" has no prompt-input category`) {
+		t.Fatalf("Build() error = %v, want uncategorized stanza error", err)
+	}
+}
+
+func TestLoadYAMLRejectsMisplacedStanza(t *testing.T) {
+	data := []byte(`
+schema_version: weatherreporter.data_package.v2
+run_id: 20260529T100000Z_daily_today
+report:
+  id: daily_today
+  prompt_id: weather.daily_report
+  generated_at: 2026-05-29T10:00:00Z
+  timezone: America/Chicago
+  current_local_date: "2026-05-29"
+  valid_period:
+    start: 2026-05-29T05:00:00Z
+    end: 2026-05-30T05:00:00Z
+briefing:
+  metadata:
+    run_id: 20260529T100000Z_daily_today
+  raw_data:
+    alert_digest:
+      checked: true
+recent_changes:
+  items: []
+`)
+
+	_, err := LoadYAML(data)
+	if err == nil || !strings.Contains(err.Error(), `briefing stanza "alert_digest" belongs under category "applicable_risk_products"`) {
+		t.Fatalf("LoadYAML() error = %v, want misplaced stanza error", err)
 	}
 }
 
@@ -190,6 +250,8 @@ func validBuildRequest(t *testing.T) BuildRequest {
 			module.Output{ID: module.Metadata, StanzaName: "metadata", Value: map[string]string{"run_id": "20260529T100000Z_daily_today"}},
 			module.Output{ID: module.CurrentConditions, StanzaName: "current_conditions", Value: map[string]string{"condition_text": "Partly cloudy"}},
 			module.Output{ID: module.DerivedDailySummary, StanzaName: "derived_daily_summary", Value: map[string]string{"date": "2026-05-29"}},
+			module.Output{ID: module.AlertDigest, StanzaName: "alert_digest", Value: map[string]bool{"checked": true}},
+			module.Output{ID: module.NarrativeForecast, StanzaName: "narrative_forecast", Value: map[string]string{"product": "narrative"}},
 		),
 	}
 }