package promptassets_test import ( "context" "encoding/json" "io/fs" "regexp" "strings" "testing" "time" "gitea.maximumdirect.net/eric/promptkit" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptassets" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gopkg.in/yaml.v3" ) type promptDefinition struct { ID string `yaml:"id"` Version string `yaml:"version"` DefaultProfile string `yaml:"default_profile"` Messages []struct { ContentFile string `yaml:"content_file"` } `yaml:"messages"` Inputs []struct { Name string `yaml:"name"` Required bool `yaml:"required"` ContentType string `yaml:"content_type"` } `yaml:"inputs"` Output struct { Format string `yaml:"format"` ValidationMode string `yaml:"validation_mode"` SchemaPath string `yaml:"schema_path"` RepairAttempts *int `yaml:"repair_attempts"` } `yaml:"output"` } func TestPromptAssetsDeclareTheFourGeneratedTextPrompts(t *testing.T) { tests := []struct { path string id string schemaID string profile string }{ {"daily/daily_generated_text.yml", "weather.daily_generated_text", "daily", "weather-balanced"}, {"today/today_generated_text.yml", "weather.today_generated_text", "today", "weather-balanced"}, {"tomorrow/tomorrow_generated_text.yml", "weather.tomorrow_generated_text", "tomorrow", "weather-balanced"}, {"hourly/hourly_generated_text.yml", "weather.hourly_generated_text", "hourly", "weather-light"}, } definitions := 0 if err := fs.WalkDir(promptassets.PromptFS(), ".", func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } if !entry.IsDir() && strings.HasSuffix(path, ".yml") { definitions++ } return nil }); err != nil { t.Fatalf("walk embedded prompts: %v", err) } if definitions != len(tests) { t.Fatalf("prompt definitions = %d, want %d", definitions, len(tests)) } for _, tc := range tests { t.Run(tc.id, func(t *testing.T) { data, err := fs.ReadFile(promptassets.PromptFS(), tc.path) if err != nil { t.Fatalf("read prompt definition: %v", err) } var definition promptDefinition if err := yaml.Unmarshal(data, &definition); err != nil { t.Fatalf("decode prompt definition: %v", err) } if definition.ID != tc.id || definition.Version != "2.0.0" || definition.DefaultProfile != tc.profile { t.Fatalf("definition = %#v, want %s version 2.0.0 and profile %s", definition, tc.id, tc.profile) } sharedInstruction := false for _, message := range definition.Messages { if message.ContentFile == "../common/data_package.user.md" { sharedInstruction = true } } if !sharedInstruction { t.Fatalf("definition messages = %#v, want shared data-package instruction", definition.Messages) } if len(definition.Inputs) != 1 || definition.Inputs[0].Name != "data_package" || !definition.Inputs[0].Required || definition.Inputs[0].ContentType != "application/yaml" { t.Fatalf("inputs = %#v, want one required YAML data_package", definition.Inputs) } if definition.Output.Format != "json" || definition.Output.ValidationMode != "json_schema" || definition.Output.SchemaPath != tc.schemaID+".generated_text.schema.json" || definition.Output.RepairAttempts != nil { t.Fatalf("output = %#v, want JSON schema output without repair attempts", definition.Output) } if _, err := promptassets.Schema(tc.schemaID); err != nil { t.Fatalf("Schema(%q) error = %v", tc.schemaID, err) } }) } } func TestSharedPromptReferencesSerializedPathsAndPreservesHazardLocality(t *testing.T) { sharedPrompt, err := fs.ReadFile(promptassets.PromptFS(), "common/data_package.user.md") if err != nil { t.Fatalf("read shared prompt: %v", err) } pkgYAML := representativeDataPackageYAML(t) paths := regexp.MustCompile(`briefing(?:\.[a-z_]+)+`).FindAllString(string(sharedPrompt), -1) if len(paths) == 0 { t.Fatal("shared prompt does not reference briefing paths") } for _, path := range paths { if !yamlPathExists(t, pkgYAML, path) { t.Fatalf("shared prompt references path %q that is absent from representative data package:\n%s", path, pkgYAML) } } for _, obsoletePath := range []string{ "briefing.metadata.alerts", "briefing.derived_daily_summary", "briefing.derived_daypart_summaries", "briefing.precip_timing", "briefing.outdoor_windows", } { if strings.Contains(string(sharedPrompt), obsoletePath) { t.Fatalf("shared prompt references obsolete path %q", obsoletePath) } } for _, requiredGuidance := range []string{ "location-matched local conclusions", "regional context unless their own geography establishes point relevance", "never present them as point-local hazards solely because they are included", } { if !strings.Contains(string(sharedPrompt), requiredGuidance) { t.Fatalf("shared prompt is missing hazard-locality guidance %q", requiredGuidance) } } } func representativeDataPackageYAML(t *testing.T) []byte { t.Helper() generatedAt := time.Date(2026, 5, 29, 10, 0, 0, 0, time.UTC) snapshot, err := module.NewSnapshot([]module.Output{ {ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"location": "Testville"}}, {ID: module.AlertDigest, StanzaName: "alert_digest", Value: map[string]any{"relevant_count": 1}}, {ID: module.SPCConvectiveOutlooks, StanzaName: "spc_convective_outlooks", Value: map[string]any{"risk_digest": []any{}, "outlooks": []any{}}}, {ID: module.DerivedDailySummary, StanzaName: "derived_daily_summary", Value: map[string]any{"theme": "dry"}}, {ID: module.DerivedDaypartSummaries, StanzaName: "derived_daypart_summaries", Value: map[string]any{"dayparts": []any{}}}, {ID: module.PrecipTiming, StanzaName: "precip_timing", Value: map[string]any{"maximum_probability": 0}}, {ID: module.OutdoorWindows, StanzaName: "outdoor_windows", Value: map[string]any{"windows": []any{}}}, {ID: module.NarrativeForecast, StanzaName: "narrative_forecast", Value: map[string]any{"periods": []any{}}}, {ID: module.AreaForecastDiscussion, StanzaName: "area_forecast_discussion", Value: map[string]any{"key_messages": []any{}, "short_term": "", "long_term": ""}}, {ID: module.SPCConvectiveDiscussion, StanzaName: "spc_convective_discussion", Value: map[string]any{"discussions": []any{}}}, {ID: module.WeatherStory, StanzaName: "weather_story", Value: map[string]any{}}, {ID: module.CurrentConditions, StanzaName: "current_conditions", Value: map[string]any{"temperature": 72}}, {ID: module.HourlyForecast, StanzaName: "hourly_forecast", Value: map[string]any{"periods": []any{}}}, }) if err != nil { t.Fatalf("NewSnapshot() error = %v", err) } pkg, err := promptinput.Build(promptinput.BuildRequest{ Metadata: promptinput.Metadata{ RunID: "20260529T100000Z_daily", ReportID: report.Daily, PromptID: "weather.daily_generated_text", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, }, Modules: snapshot, }) if err != nil { t.Fatalf("Build() error = %v", err) } data, err := promptinput.MarshalYAML(pkg) if err != nil { t.Fatalf("MarshalYAML() error = %v", err) } return data } func yamlPathExists(t *testing.T, data []byte, path string) bool { t.Helper() var document map[string]any if err := yaml.Unmarshal(data, &document); err != nil { t.Fatalf("decode representative data package: %v", err) } var current any = document for _, segment := range strings.Split(path, ".") { mapping, ok := current.(map[string]any) if !ok { return false } current, ok = mapping[segment] if !ok { return false } } return true } func TestSchemasAreCanonicalAndIndependent(t *testing.T) { for _, id := range []string{"daily", "today", "tomorrow", "hourly"} { t.Run(id, func(t *testing.T) { data, err := promptassets.Schema(id) if err != nil { t.Fatalf("Schema() error = %v", err) } var schema struct { ID string `json:"$id"` Title string `json:"title"` Type string `json:"type"` AdditionalProperties bool `json:"additionalProperties"` Required []string `json:"required"` Properties map[string]any `json:"properties"` } if err := json.Unmarshal(data, &schema); err != nil { t.Fatalf("decode schema: %v", err) } if schema.Type != "object" || schema.AdditionalProperties || strings.Join(schema.Required, ",") != "summary,forecast_discussion,precipitation_timing" { t.Fatalf("schema = %#v, want strict generated-text object", schema) } assertGeneratedTextSchemaShape(t, id, schema.Properties) if _, ok := schema.Properties["confidence"]; ok { t.Fatalf("schema properties = %#v, do not want retired confidence field", schema.Properties) } if id == "daily" && (schema.ID != "weatherreporter.daily.generated_text.schema.json" || schema.Title != "Daily GeneratedText") { t.Fatalf("daily schema identity = %q/%q, want corrected Daily identity", schema.ID, schema.Title) } data[0] = 'x' fresh, err := promptassets.Schema(id) if err != nil || fresh[0] != '{' { t.Fatalf("Schema() returned shared data or error: %v", err) } }) } } func assertGeneratedTextSchemaShape(t *testing.T, id string, properties map[string]any) { t.Helper() for _, name := range []string{"summary", "precipitation_timing"} { property, ok := properties[name].(map[string]any) if !ok || property["type"] != "string" { t.Fatalf("schema property %q = %#v, want string", name, properties[name]) } } discussion, ok := properties["forecast_discussion"].(map[string]any) if !ok { t.Fatalf("schema forecast_discussion = %#v, want property", properties["forecast_discussion"]) } if id == "hourly" { if discussion["type"] != "string" || discussion["maxLength"] != float64(12_000) { t.Fatalf("hourly forecast_discussion = %#v, want bounded string", discussion) } return } if discussion["type"] != "array" || discussion["minItems"] != float64(1) || discussion["maxItems"] != float64(12) { t.Fatalf("%s forecast_discussion = %#v, want bounded non-empty array", id, discussion) } items, ok := discussion["items"].(map[string]any) if !ok || items["type"] != "string" || items["maxLength"] != float64(4_000) { t.Fatalf("%s forecast_discussion items = %#v, want bounded strings", id, discussion["items"]) } } func TestPromptkitInspectsEmbeddedPromptsOffline(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(promptassets.PromptFS(), "."), promptkit.WithSchemaFS(promptassets.SchemaFS(), "."), promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."), ) if err != nil { t.Fatalf("NewEngine() error = %v", err) } for _, want := range []struct { id string profile string model string }{ {"weather.daily_generated_text", "weather-balanced", "~google/gemini-flash-latest"}, {"weather.today_generated_text", "weather-balanced", "~google/gemini-flash-latest"}, {"weather.tomorrow_generated_text", "weather-balanced", "~google/gemini-flash-latest"}, {"weather.hourly_generated_text", "weather-light", "deepseek/deepseek-v4-flash"}, } { t.Run(want.id, func(t *testing.T) { inspection, err := engine.InspectPrompt(context.Background(), want.id, "2.0.0") if err != nil { t.Fatalf("InspectPrompt() error = %v", err) } if inspection.PromptID != want.id || inspection.PromptVersion != "2.0.0" || inspection.DefaultProfileID != want.profile { t.Fatalf("inspection = %#v", inspection) } profile, err := engine.InspectProfile(context.Background(), inspection.DefaultProfileID) if err != nil || profile.EffectiveModelParams.Model != want.model { t.Fatalf("profile/error = %#v/%v, want model %q", profile, err, want.model) } }) } profile, err := engine.InspectProfile(context.Background(), "weather-deep") if err != nil || profile.EffectiveModelParams.Model != "~anthropic/claude-sonnet-latest" { t.Fatalf("weather-deep profile/error = %#v/%v", profile, err) } } func TestEmbeddedProfilesAreCompleteAndInspectable(t *testing.T) { wantPaths := map[string]bool{ "weather-balanced.yml": false, "weather-deep.yml": false, "weather-light.yml": false, } if err := fs.WalkDir(promptassets.ProfileFS(), ".", func(path string, entry fs.DirEntry, err error) error { if err != nil || entry.IsDir() { return err } if _, ok := wantPaths[path]; !ok { t.Fatalf("unexpected embedded profile asset %q", path) } wantPaths[path] = true return nil }); err != nil { t.Fatalf("walk embedded profiles: %v", err) } for path, found := range wantPaths { if !found { t.Errorf("missing embedded profile asset %q", path) } } engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(promptassets.PromptFS(), "."), promptkit.WithSchemaFS(promptassets.SchemaFS(), "."), promptkit.WithFallbackProfileFS(promptassets.ProfileFS(), "."), ) if err != nil { t.Fatalf("NewEngine() error = %v", err) } profiles := []struct { id string model string timeoutSeconds int reasoningEffort string }{ {"weather-light", "deepseek/deepseek-v4-flash", 180, ""}, {"weather-balanced", "~google/gemini-flash-latest", 240, "high"}, {"weather-deep", "~anthropic/claude-sonnet-latest", 240, "high"}, } for _, want := range profiles { t.Run(want.id, func(t *testing.T) { inspection, err := engine.InspectProfile(context.Background(), want.id) if err != nil { t.Fatalf("InspectProfile() error = %v", err) } got := inspection.EffectiveModelParams if inspection.ProfileID != want.id || got.BackendID != "openrouter" || got.Model != want.model || got.TimeoutSeconds != want.timeoutSeconds || got.ServiceTier != "flex" || got.ReasoningEffort != want.reasoningEffort { t.Fatalf("inspection = %#v, want %q using openrouter model %q", inspection, want.id, want.model) } }) } } func TestEmbeddedProfilesAreMinimalBaseAliases(t *testing.T) { wantBases := map[string]string{ "weather-balanced.yml": "gemini-flash-latest", "weather-deep.yml": "claude-sonnet-latest", "weather-light.yml": "deepseek-4-flash", } for path, wantBase := range wantBases { t.Run(path, func(t *testing.T) { data, err := fs.ReadFile(promptassets.ProfileFS(), path) if err != nil { t.Fatalf("read profile: %v", err) } var definition map[string]string if err := yaml.Unmarshal(data, &definition); err != nil { t.Fatalf("unmarshal profile: %v", err) } if definition["id"] != strings.TrimSuffix(path, ".yml") || definition["base_profile"] != wantBase || len(definition) != 2 { t.Fatalf("profile definition = %#v, want only its id and base profile %q", definition, wantBase) } }) } } func TestEmbeddedProfilesExcludeUnsafeOrIncidentalSettings(t *testing.T) { forbidden := []string{"endpoint:", "api_key", "credential", "temperature:", "top_p:", "max_tokens:"} if err := fs.WalkDir(promptassets.ProfileFS(), ".", func(path string, entry fs.DirEntry, err error) error { if err != nil || entry.IsDir() { return err } data, err := fs.ReadFile(promptassets.ProfileFS(), path) if err != nil { return err } for _, setting := range forbidden { if strings.Contains(string(data), setting) { t.Fatalf("%s contains forbidden profile setting %q", path, setting) } } return nil }); err != nil { t.Fatalf("walk embedded profiles: %v", err) } } func TestPromptAssetsExcludeRetiredRuntimeSettings(t *testing.T) { if err := fs.WalkDir(promptassets.PromptFS(), ".", func(path string, entry fs.DirEntry, err error) error { if err != nil || entry.IsDir() { return err } data, err := fs.ReadFile(promptassets.PromptFS(), path) if err != nil { return err } for _, unwanted := range []string{"local-heavy", "pipeline-weather/", "application/json", "repair_attempts:", "weather.daily_report"} { if strings.Contains(string(data), unwanted) { t.Fatalf("%s contains retired runtime setting %q", path, unwanted) } } return nil }); err != nil { t.Fatalf("walk embedded prompts: %v", err) } }