eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactor the yaml data package to group data sources by category

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-10 11:50:22 -05:00
parent 276e4f1189
commit f149563c68
4 changed files with 256 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
docs/internal/prompt-input.md
View File

@@ -24,7 +24,8 @@ Inputs:
Outputs:
- `promptinput.Package` with schema version, RunID, report metadata, named
module stanzas, Recent Changes, and source warnings
module stanzas grouped for prompt presentation, Recent Changes, and source
warnings
- YAML bytes from `promptinput.MarshalYAML`
- YAML file written atomically by `promptinput.Save`
@@ -38,15 +39,37 @@ report:
prompt_id: <prompt_id>
briefing:
metadata: {}
current_conditions: {}
narrative_forecast: {}
hourly_forecast: {}
applicable_risk_products:
alert_digest: {}
derived_summaries:
derived_daily_summary: {}
derived_daypart_summaries: {}
precip_timing: {}
outdoor_windows: {}
narrative_products:
narrative_forecast: {}
area_forecast_discussion: {}
weather_story: {}
raw_data:
current_conditions: {}
hourly_forecast: {}
recent_changes:
items: []
```
The `briefing` mapping contains named module stanzas. Stanza order follows the
module snapshot output order.
The `briefing` mapping keeps `metadata` directly under `briefing` and groups
weather module stanzas under prompt-facing categories. This grouping is a YAML
presentation concern only: module snapshots remain flat, and loaded
`promptinput.Package` values expose flat stanza names in `Briefing.Values`.
Within each category, stanza order follows the module snapshot output order.
Current categories are:
- `applicable_risk_products`: location-applicable alerts, warnings, outlooks,
discussions, and similar risk products.
- `derived_summaries`: deterministic summaries and calculated report facts.
- `narrative_products`: official narrative text products and forecast stories.
- `raw_data`: minimally transformed underlying weather data.
## Boundaries
@@ -92,6 +115,7 @@ Inspect:
## Invariants
- Scriptorium receives structured YAML through `--input data_package=<path>`.
- Module stanza order is deterministic for generated snapshots.
- Module stanza order is deterministic within each prompt-facing category.
- Every non-metadata module stanza has exactly one prompt-input category.
- Recent Changes are provided by `internal/changes`; this package does not
infer changes from rendered report text.

24
internal/app/app_test.go
View File

@@ -160,6 +160,10 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) {
}
if !strings.Contains(string(data), "schema_version: weatherreporter.data_package.v2") ||
!strings.Contains(string(data), "recent_changes:") ||
!strings.Contains(string(data), "applicable_risk_products:") ||
!strings.Contains(string(data), "derived_summaries:") ||
!strings.Contains(string(data), "narrative_products:") ||
!strings.Contains(string(data), "raw_data:") ||
!strings.Contains(string(data), "current_conditions:") ||
!strings.Contains(string(data), "narrative_forecast:") ||
!strings.Contains(string(data), "hourly_forecast:") ||
@@ -169,13 +173,19 @@ func TestGenerateReportWritesReportAndPreflight(t *testing.T) {
if strings.Contains(string(data), "source_warnings:") {
t.Fatalf("data package has source warnings, want none for complete fetched sources:\n%s", string(data))
}
currentIndex := strings.Index(string(data), " current_conditions:")
narrativeIndex := strings.Index(string(data), " narrative_forecast:")
summaryIndex := strings.Index(string(data), " derived_daily_summary:")
outdoorIndex := strings.Index(string(data), " outdoor_windows:")
hourlyIndex := strings.Index(string(data), " hourly_forecast:")
if currentIndex < 0 || narrativeIndex < 0 || summaryIndex < 0 || outdoorIndex < 0 || hourlyIndex < 0 || !(currentIndex < narrativeIndex && narrativeIndex < summaryIndex && outdoorIndex < hourlyIndex) {
t.Fatalf("data package stanza order is wrong, want current_conditions then narrative_forecast then derived_daily_summary and final hourly_forecast:\n%s", string(data))
riskIndex := strings.Index(string(data), " applicable_risk_products:")
derivedIndex := strings.Index(string(data), " derived_summaries:")
narrativeIndex := strings.Index(string(data), " narrative_products:")
rawIndex := strings.Index(string(data), " raw_data:")
alertIndex := strings.Index(string(data), " alert_digest:")
summaryIndex := strings.Index(string(data), " derived_daily_summary:")
storyIndex := strings.Index(string(data), " weather_story:")
currentIndex := strings.Index(string(data), " current_conditions:")
hourlyIndex := strings.Index(string(data), " hourly_forecast:")
if riskIndex < 0 || derivedIndex < 0 || narrativeIndex < 0 || rawIndex < 0 || alertIndex < 0 || summaryIndex < 0 || storyIndex < 0 || currentIndex < 0 || hourlyIndex < 0 ||
!(riskIndex < derivedIndex && derivedIndex < narrativeIndex && narrativeIndex < rawIndex) ||
!(riskIndex < alertIndex && derivedIndex < summaryIndex && narrativeIndex < storyIndex && rawIndex < currentIndex && currentIndex < hourlyIndex) {
t.Fatalf("data package grouping is wrong, want categorized prompt stanzas:\n%s", string(data))
}
savedDataPackage, err := promptinput.LoadYAML(data)
if err != nil {

148
internal/promptinput/package.go
View File

@@ -18,6 +18,35 @@ import (
const SchemaVersion = "weatherreporter.data_package.v2"
const (
metadataStanza = "metadata"
categoryApplicableRiskProducts = "applicable_risk_products"
categoryDerivedSummaries = "derived_summaries"
categoryNarrativeProducts = "narrative_products"
categoryRawData = "raw_data"
)
var briefingCategoryOrder = []string{
categoryApplicableRiskProducts,
categoryDerivedSummaries,
categoryNarrativeProducts,
categoryRawData,
}
var briefingStanzaCategories = map[string]string{
string(module.AlertDigest): categoryApplicableRiskProducts,
string(module.DerivedDailySummary): categoryDerivedSummaries,
string(module.DerivedDaypartSummaries): categoryDerivedSummaries,
string(module.PrecipTiming): categoryDerivedSummaries,
string(module.OutdoorWindows): categoryDerivedSummaries,
string(module.TomorrowPlanning): categoryDerivedSummaries,
string(module.NarrativeForecast): categoryNarrativeProducts,
string(module.AreaForecastDiscussion): categoryNarrativeProducts,
string(module.WeatherStory): categoryNarrativeProducts,
string(module.CurrentConditions): categoryRawData,
string(module.HourlyForecast): categoryRawData,
}
type BuildRequest struct {
Metadata Metadata
Modules module.Snapshot
@@ -144,13 +173,24 @@ func Validate(pkg Package) error {
if len(pkg.Briefing.Order) == 0 {
return fmt.Errorf("briefing stanzas are required")
}
seen := map[string]struct{}{}
for _, name := range pkg.Briefing.Order {
if name == "" {
return fmt.Errorf("briefing stanza name is required")
}
if _, ok := seen[name]; ok {
return fmt.Errorf("duplicate briefing stanza %q", name)
}
seen[name] = struct{}{}
if _, ok := pkg.Briefing.Values[name]; !ok {
return fmt.Errorf("briefing stanza %q is missing", name)
}
if name == metadataStanza {
continue
}
if _, ok := briefingStanzaCategories[name]; !ok {
return fmt.Errorf("briefing stanza %q has no prompt-input category", name)
}
}
return nil
}
@@ -191,17 +231,40 @@ func LoadYAML(data []byte) (Package, error) {
func (b BriefingStanzas) MarshalYAML() (any, error) {
node := &yaml.Node{Kind: yaml.MappingNode}
categoryNames := map[string][]string{}
for _, name := range b.Order {
value, ok := b.Values[name]
if !ok {
continue
}
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: name}
valueNode, err := yamlNode(value)
if err != nil {
return nil, fmt.Errorf("marshal briefing stanza %q: %w", name, err)
if name == metadataStanza {
if err := appendYAMLMappingValue(node, name, value); err != nil {
return nil, err
}
continue
}
node.Content = append(node.Content, keyNode, valueNode)
category, ok := briefingStanzaCategories[name]
if !ok {
return nil, fmt.Errorf("briefing stanza %q has no prompt-input category", name)
}
categoryNames[category] = append(categoryNames[category], name)
}
for _, category := range briefingCategoryOrder {
names := categoryNames[category]
if len(names) == 0 {
continue
}
categoryNode := &yaml.Node{Kind: yaml.MappingNode}
for _, name := range names {
value := b.Values[name]
if err := appendYAMLMappingValue(categoryNode, name, value); err != nil {
return nil, err
}
}
node.Content = append(node.Content,
&yaml.Node{Kind: yaml.ScalarNode, Value: category},
categoryNode,
)
}
return node, nil
}
@@ -212,14 +275,46 @@ func (b *BriefingStanzas) UnmarshalYAML(value *yaml.Node) error {
}
values := map[string]any{}
order := make([]string, 0, len(value.Content)/2)
seen := map[string]struct{}{}
seenCategories := map[string]struct{}{}
categoryOrder := map[string][]string{}
for i := 0; i < len(value.Content); i += 2 {
name := value.Content[i].Value
var stanza any
if err := value.Content[i+1].Decode(&stanza); err != nil {
return err
if name == metadataStanza {
if err := decodeBriefingStanza(value.Content[i+1], name, values, &order, seen); err != nil {
return err
}
continue
}
order = append(order, name)
values[name] = stanza
if !knownBriefingCategory(name) {
return fmt.Errorf("unknown briefing category %q", name)
}
if _, ok := seenCategories[name]; ok {
return fmt.Errorf("duplicate briefing category %q", name)
}
seenCategories[name] = struct{}{}
categoryNode := value.Content[i+1]
if categoryNode.Kind != yaml.MappingNode {
return fmt.Errorf("briefing category %q must be a mapping", name)
}
var names []string
for j := 0; j < len(categoryNode.Content); j += 2 {
stanzaName := categoryNode.Content[j].Value
category, ok := briefingStanzaCategories[stanzaName]
if !ok {
return fmt.Errorf("briefing stanza %q has no prompt-input category", stanzaName)
}
if category != name {
return fmt.Errorf("briefing stanza %q belongs under category %q, not %q", stanzaName, category, name)
}
if err := decodeBriefingStanza(categoryNode.Content[j+1], stanzaName, values, &names, seen); err != nil {
return err
}
}
categoryOrder[name] = names
}
for _, category := range briefingCategoryOrder {
order = append(order, categoryOrder[category]...)
}
b.Order = order
b.Values = values
@@ -250,6 +345,39 @@ func (b *BriefingStanzas) UnmarshalJSON(data []byte) error {
return nil
}
func appendYAMLMappingValue(node *yaml.Node, name string, value any) error {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: name}
valueNode, err := yamlNode(value)
if err != nil {
return fmt.Errorf("marshal briefing stanza %q: %w", name, err)
}
node.Content = append(node.Content, keyNode, valueNode)
return nil
}
func decodeBriefingStanza(node *yaml.Node, name string, values map[string]any, order *[]string, seen map[string]struct{}) error {
if _, ok := seen[name]; ok {
return fmt.Errorf("duplicate briefing stanza %q", name)
}
var stanza any
if err := node.Decode(&stanza); err != nil {
return err
}
seen[name] = struct{}{}
*order = append(*order, name)
values[name] = stanza
return nil
}
func knownBriefingCategory(name string) bool {
for _, category := range briefingCategoryOrder {
if name == category {
return true
}
}
return false
}
func yamlNode(value any) (*yaml.Node, error) {
data, err := json.Marshal(value)
if err != nil {

78
internal/promptinput/package_test.go
View File

@@ -106,7 +106,7 @@ func TestBuildUsesNamedSnapshotStanzas(t *testing.T) {
req.Metadata.PromptID = "weather.three_day_outlook"
req.Modules = snapshotWithOutputs(t,
module.Output{ID: module.Metadata, StanzaName: "metadata", Value: map[string]string{"run_id": req.Metadata.RunID}},
module.Output{ID: module.DerivedDaypartSummaries, StanzaName: "three_day", Value: map[string]any{"days": []string{"2026-05-29"}}},
module.Output{ID: module.DerivedDaypartSummaries, StanzaName: "derived_daypart_summaries", Value: map[string]any{"days": []string{"2026-05-29"}}},
)
pkg, err := Build(req)
@@ -117,12 +117,12 @@ func TestBuildUsesNamedSnapshotStanzas(t *testing.T) {
if pkg.Report.ID != report.ThreeDay {
t.Fatalf("Report.ID = %q, want three_day", pkg.Report.ID)
}
if _, ok := pkg.Briefing.Values["three_day"]; !ok {
t.Fatal("Briefing.Values[three_day] missing")
if _, ok := pkg.Briefing.Values["derived_daypart_summaries"]; !ok {
t.Fatal("Briefing.Values[derived_daypart_summaries] missing")
}
}
func TestMarshalYAMLIsDeterministicAndUsesNamedStanzas(t *testing.T) {
func TestMarshalYAMLIsDeterministicAndGroupsNamedStanzas(t *testing.T) {
pkg, err := Build(validBuildRequest(t))
if err != nil {
t.Fatalf("Build() error = %v", err)
@@ -141,9 +141,26 @@ func TestMarshalYAMLIsDeterministicAndUsesNamedStanzas(t *testing.T) {
}
if !strings.Contains(string(first), "schema_version: weatherreporter.data_package.v2") ||
!strings.Contains(string(first), "briefing:\n") ||
!strings.Contains(string(first), " applicable_risk_products:\n") ||
!strings.Contains(string(first), " derived_summaries:\n") ||
!strings.Contains(string(first), " narrative_products:\n") ||
!strings.Contains(string(first), " raw_data:\n") ||
!strings.Contains(string(first), " current_conditions:\n") ||
!strings.Contains(string(first), " condition_text: Partly cloudy") {
t.Fatalf("YAML output missing expected named stanzas:\n%s", string(first))
!strings.Contains(string(first), " condition_text: Partly cloudy") {
t.Fatalf("YAML output missing expected grouped stanzas:\n%s", string(first))
}
for _, pair := range []struct {
before string
after string
}{
{before: " metadata:\n", after: " applicable_risk_products:\n"},
{before: " applicable_risk_products:\n", after: " derived_summaries:\n"},
{before: " derived_summaries:\n", after: " narrative_products:\n"},
{before: " narrative_products:\n", after: " raw_data:\n"},
} {
if strings.Index(string(first), pair.before) < 0 || strings.Index(string(first), pair.after) < 0 || strings.Index(string(first), pair.before) > strings.Index(string(first), pair.after) {
t.Fatalf("YAML category order is wrong, want %q before %q:\n%s", pair.before, pair.after, string(first))
}
}
}
@@ -165,8 +182,51 @@ func TestLoadYAMLRoundTrip(t *testing.T) {
if loaded.SchemaVersion != SchemaVersion || loaded.RunID != pkg.RunID {
t.Fatalf("loaded package = %#v, want schema and run id", loaded)
}
if loaded.Briefing.Order[1] != "current_conditions" {
t.Fatalf("loaded package order = %#v, want current_conditions second", loaded.Briefing.Order)
wantOrder := []string{"metadata", "alert_digest", "derived_daily_summary", "narrative_forecast", "current_conditions"}
if strings.Join(loaded.Briefing.Order, ",") != strings.Join(wantOrder, ",") {
t.Fatalf("loaded package order = %#v, want grouped category order %#v", loaded.Briefing.Order, wantOrder)
}
if got := loaded.Briefing.Values["current_conditions"].(map[string]any)["condition_text"]; got != "Partly cloudy" {
t.Fatalf("loaded current_conditions.condition_text = %#v, want Partly cloudy", got)
}
}
func TestMarshalYAMLRejectsUncategorizedStanza(t *testing.T) {
req := validBuildRequest(t)
req.Modules = snapshotWithOutputs(t, module.Output{ID: module.ID("custom"), StanzaName: "custom", Value: map[string]string{"value": "x"}})
_, err := Build(req)
if err == nil || !strings.Contains(err.Error(), `briefing stanza "custom" has no prompt-input category`) {
t.Fatalf("Build() error = %v, want uncategorized stanza error", err)
}
}
func TestLoadYAMLRejectsMisplacedStanza(t *testing.T) {
data := []byte(`
schema_version: weatherreporter.data_package.v2
run_id: 20260529T100000Z_daily_today
report:
id: daily_today
prompt_id: weather.daily_report
generated_at: 2026-05-29T10:00:00Z
timezone: America/Chicago
current_local_date: "2026-05-29"
valid_period:
start: 2026-05-29T05:00:00Z
end: 2026-05-30T05:00:00Z
briefing:
metadata:
run_id: 20260529T100000Z_daily_today
raw_data:
alert_digest:
checked: true
recent_changes:
items: []
`)
_, err := LoadYAML(data)
if err == nil || !strings.Contains(err.Error(), `briefing stanza "alert_digest" belongs under category "applicable_risk_products"`) {
t.Fatalf("LoadYAML() error = %v, want misplaced stanza error", err)
}
}
@@ -190,6 +250,8 @@ func validBuildRequest(t *testing.T) BuildRequest {
module.Output{ID: module.Metadata, StanzaName: "metadata", Value: map[string]string{"run_id": "20260529T100000Z_daily_today"}},
module.Output{ID: module.CurrentConditions, StanzaName: "current_conditions", Value: map[string]string{"condition_text": "Partly cloudy"}},
module.Output{ID: module.DerivedDailySummary, StanzaName: "derived_daily_summary", Value: map[string]string{"date": "2026-05-29"}},
module.Output{ID: module.AlertDigest, StanzaName: "alert_digest", Value: map[string]bool{"checked": true}},
module.Output{ID: module.NarrativeForecast, StanzaName: "narrative_forecast", Value: map[string]string{"product": "narrative"}},
),
}
}