eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Run report generation without workspace state

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-01 19:52:22 +00:00
parent 4bdba6f2b7
commit 7ffc3dc603
21 changed files with 343 additions and 3464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
internal/cli/result.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
@@ -17,26 +18,25 @@ const (
)
type generateSummary struct {
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"`
GeneratedTextPath string `json:"generatedTextPath,omitempty"`
RenderContextPath string `json:"renderContextPath,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
OutputPath string `json:"outputPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
PromptVersion string `json:"promptVersion"`
Timezone string `json:"timezone"`
ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus string `json:"validationStatus,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
}
type generateNotificationSummary struct {
@@ -73,24 +73,20 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
return summary
}
metadata := result.Metadata
summary.ReportID = metadata.ReportID
summary.ReportName = reportName(metadata.ReportID)
summary.PromptID = metadata.PromptID
summary.RunID = metadata.RunID
summary.ReportID = result.ReportID
summary.ReportName = result.ReportName
summary.PromptID = result.PromptID
summary.PromptVersion = result.PromptVersion
summary.RunID = result.RunID
summary.Status = summaryStatusSucceeded
summary.GeneratedAt = metadata.GeneratedAt
summary.ValidPeriod = metadata.ValidPeriod
summary.ReportPath = result.ReportPath
summary.GeneratedAt = result.GeneratedAt
summary.ValidPeriod = result.ValidPeriod
summary.Timezone = result.Timezone
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
summary.ValidationStatus = string(result.ValidationStatus)
summary.OutputPath = result.OutputPath
summary.MetadataPath = result.MetadataPath
summary.DataPackagePath = result.DataPackagePath
summary.PreparationPath = result.PreparationPath
summary.ExecutionPath = result.ExecutionPath
summary.LLMDebugPath = result.LLMDebugPath
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
summary.GeneratedTextPath = result.GeneratedTextPath
summary.RenderContextPath = result.RenderContextPath
summary.Notification = newGenerateNotificationSummary(result.Notification)
if err != nil {
summary.Status = summaryStatusFailed

251
internal/cli/result_test.go
View File

@@ -2,264 +2,29 @@ package cli
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
acceptedAt := generatedAt.Add(time.Minute)
startedAt := acceptedAt.Add(time.Minute)
finishedAt := startedAt.Add(time.Minute)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json",
GeneratedTextPath: "/runs/hourly/generated_text.json",
RenderContextPath: "/runs/hourly/render_context.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
Notification: &app.NotificationResult{
Status: "succeeded",
UploadStatus: "accepted",
RunID: "distributor-run",
PipelineID: "weatherreporter.hourly",
BundleID: "weatherreporter.home.hourly",
IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly",
AcceptedAt: acceptedAt,
StartedAt: &startedAt,
FinishedAt: &finishedAt,
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
},
}
summary := newGenerateSummary(result, nil)
if summary.Command != "generate" || summary.Status != "succeeded" {
t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status)
}
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {
t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification)
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") {
t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data))
}
if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") {
t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data))
}
}
func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
ReportPath: "/runs/daily/report.md",
OutputPath: "/copies/daily.md",
MetadataPath: "/runs/daily/metadata.json",
Metadata: state.Metadata{
ReportID: report.Daily,
PromptID: "weather.daily_generated_text",
RunID: "20260529T133000Z_daily",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
summary := newGenerateSummary(result, nil)
if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" {
t.Fatalf("summary = %#v, want successful daily summary", summary)
}
if summary.Notification != nil {
t.Fatalf("notification summary = %#v, want omitted", summary.Notification)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
for _, omitted := range []string{"notification"} {
if strings.Contains(string(data), omitted) {
t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data))
for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
if strings.Contains(string(data), forbidden) {
t.Fatalf("summary includes %q: %s", forbidden, data)
}
}
}
func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) {
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
Metadata: state.Metadata{
ReportID: report.Daily,
RunID: "20260529T133000Z_daily",
},
}
data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed")))
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("partial summary omits reached paths:\n%s", text)
}
}
func TestNewGenerateSummaryForNotificationFailure(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`)
summary := newGenerateSummary(result, err)
if summary.Status != "failed" || summary.Error != err.Error() {
t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error)
}
if summary.ReportPath == "" || summary.MetadataPath == "" {
t.Fatalf("artifact paths = report %q metadata %q, want retained output provenance", summary.ReportPath, summary.MetadataPath)
}
data, marshalErr := json.Marshal(summary)
if marshalErr != nil {
t.Fatalf("Marshal() error = %v", marshalErr)
}
if strings.Contains(string(data), "notificationPath") {
t.Fatalf("summary JSON includes a notification receipt path:\n%s", data)
}
}
func TestNewBatchSummaryStatusDerivation(t *testing.T) {
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
finishedAt := startedAt.Add(2 * time.Minute)
tests := []struct {
name string
result *app.BatchResult
wantStatus string
wantError string
}{
{
name: "success",
result: &app.BatchResult{
Batch: app.BatchMorning,
StartedAt: startedAt,
FinishedAt: finishedAt,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}},
},
wantStatus: "succeeded",
},
{
name: "report failure",
result: &app.BatchResult{
Batch: app.BatchMorning,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{
{ReportID: report.Today, Status: "succeeded"},
{ReportID: report.Tomorrow, Status: "failed", Error: "render failed"},
},
},
wantStatus: "failed",
wantError: "batch morning failed: 1 of 2 reports failed",
},
{
name: "skipped notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}},
Notification: &app.BatchNotificationResult{
Status: "skipped",
Reason: "one or more reports failed",
},
},
wantStatus: "failed",
wantError: "batch evening failed: 1 of 2 reports failed",
},
{
name: "failed notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}},
Notification: &app.BatchNotificationResult{
Status: "failed",
Error: "notify batch evening: upload rejected",
},
},
wantStatus: "failed",
wantError: "batch evening notification failed: notify batch evening: upload rejected",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := newBatchSummary(tt.result)
if summary.Command != "run" || summary.Status != tt.wantStatus {
t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus)
}
if summary.Error != tt.wantError {
t.Fatalf("error = %q, want %q", summary.Error, tt.wantError)
}
if len(summary.Reports) != len(tt.result.Reports) {
t.Fatalf("reports = %#v, want copied report list", summary.Reports)
}
})
}
}
func testSummaryPeriod(start time.Time) timeutil.Period {
return timeutil.Period{
Start: start,
End: start.Add(6 * time.Hour),
}
}

631
internal/cli/root_test.go
View File

@@ -1,631 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
const (
testRenderedPrompt = "PRIVATE RENDERED PROMPT"
testSchemaBody = `{"private":"schema"}`
testDataBody = "PRIVATE DATA PACKAGE"
testGeneratedBody = "PRIVATE GENERATED BODY"
testEndpoint = "https://user:credential@example.invalid/v1?token=credential"
testParameters = `{"temperature":0.2,"private":"parameter"}`
testCredential = "cli-secret-credential"
)
type commandOutput struct {
stdout string
stderr string
}
type cliExecutor struct {
fail bool
failPrompt string
}
func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text")
return promptexec.PromptInspection{
PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile",
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"},
}, nil
}
func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil
}
func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath,
StartedAt: now, EndedAt: now,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if e.fail || req.PromptID == e.failPrompt {
return nil, errors.New(strings.Join([]string{
"provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters,
}, " "))
}
raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`)
if req.PromptID == "weather.hourly_generated_text" {
raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`)
}
return &promptexec.Execution{
RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil),
}, nil
}
func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) {
output, err := runCLICommand(Runner{}, "--help")
if err != nil {
t.Fatalf("Run(--help) error = %v", err)
}
for _, command := range []string{
"--version",
"generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening",
"inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources",
} {
if !strings.Contains(output.stdout, command) {
t.Fatalf("help missing %q:\n%s", command, output.stdout)
}
}
for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} {
if strings.Contains(output.stdout, retired) {
t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout)
}
}
for _, description := range []string{"Write the generated Markdown report to PATH.", "Write generated Markdown reports beneath PATH"} {
if !strings.Contains(output.stdout, description) {
t.Fatalf("help missing output description %q:\n%s", description, output.stdout)
}
}
}
func TestRunnerVersion(t *testing.T) {
for _, test := range []struct {
name string
runner Runner
version string
}{
{name: "development default", runner: Runner{}, version: "development"},
{name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"},
} {
t.Run(test.name, func(t *testing.T) {
output, err := runCLICommand(test.runner, "--version")
if err != nil {
t.Fatalf("Run(--version) error = %v", err)
}
if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" {
t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr)
}
})
}
if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil {
t.Fatal("Run(--version extra) error = nil")
}
}
func TestResolveSupportedCommandsAndFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, constructions := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want app.ReportKind
}{
{name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily},
{name: "today", args: []string{"today"}, want: app.ReportToday},
{name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow},
{name: "hourly", args: []string{"hourly"}, want: app.ReportHourly},
} {
t.Run(tt.name, func(t *testing.T) {
args := append(tt.args, "--config", configPath)
req, err := runner.resolveGenerate(args)
if err != nil {
t.Fatalf("resolveGenerate() error = %v", err)
}
if req.Report != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want report %q with executor", req, tt.want)
}
if tt.want == app.ReportDaily || tt.want == app.ReportToday {
if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" {
t.Fatalf("resolved date = %q, want 2026-05-29", got)
}
} else if !req.Date.IsZero() {
t.Fatalf("resolved date = %s, want unset", req.Date)
}
})
}
for _, tt := range []struct {
name string
want app.BatchKind
}{
{name: "morning", want: app.BatchMorning},
{name: "evening", want: app.BatchEvening},
} {
t.Run(tt.name, func(t *testing.T) {
req, err := runner.resolveRun([]string{tt.name, "--config", configPath})
if err != nil {
t.Fatalf("resolveRun() error = %v", err)
}
if req.Batch != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want batch %q with executor", req, tt.want)
}
})
}
if *constructions != 6 {
t.Fatalf("executor constructions = %d, want one per resolved action", *constructions)
}
}
func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
generate, generateOpts, err := runner.resolveGenerateAction([]string{
"daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC",
"--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveGenerateAction() error = %v", err)
}
if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != filepath.Join(runner.WorkingDir, "daily.md") || generate.WorkingDir != runner.WorkingDir || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet {
t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts)
}
if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" {
t.Fatalf("generate date = %q, want 2026-05-30", got)
}
batch, batchOpts, err := runner.resolveRunAction([]string{
"evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports",
"--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveRunAction() error = %v", err)
}
if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != filepath.Join(runner.WorkingDir, "reports") || batch.WorkingDir != runner.WorkingDir || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet {
t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts)
}
}
func TestGenerateFlagContracts(t *testing.T) {
for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(kind), func(t *testing.T) {
opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"})
if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" {
t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err)
}
})
}
runner, _ := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want string
}{
{name: "daily requires date", args: []string{"daily"}, want: "requires --date"},
{name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}},
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}},
{name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}},
{name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}},
{name: "batch rejects output", args: []string{"run", "--out", "report.md"}},
} {
t.Run(tt.name, func(t *testing.T) {
var err error
if tt.args[0] == "run" {
_, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...))
} else {
_, err = runner.resolveGenerate(tt.args)
}
if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) {
t.Fatalf("error = %v, want rejection containing %q", err, tt.want)
}
})
}
}
func TestResolversRejectRetiredAndUnknownNames(t *testing.T) {
runner, _ := countingRunner(cliExecutor{})
for _, name := range []string{"near-term", "three-day", "weekend", "storm"} {
if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") {
t.Fatalf("resolveGenerate(%q) error = %v", name, err)
}
}
for _, name := range []string{"daily", "weekend", "storm"} {
if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") {
t.Fatalf("resolveRun(%q) error = %v", name, err)
}
}
}
func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) {
for _, tt := range []struct {
name string
args func(string, string) []string
}{
{name: "single", args: func(configPath, outputPath string) []string {
return []string{"generate", "today", "--config", configPath, "--out", outputPath}
}},
{name: "batch", args: func(configPath, outputPath string) []string {
return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath}
}},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
assertRoutineOutputSafe(t, output)
if tt.name == "single" {
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" {
t.Fatalf("single summary = %#v", summary)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout)
}
} else {
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" {
t.Fatalf("batch summary = %#v", summary)
}
if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") {
t.Fatalf("batch status = %q", output.stderr)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout)
}
}
})
}
}
func TestRunnerPreRunFailureAndQuietMode(t *testing.T) {
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "daily")
if err == nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions)
}
fixture := newCLIFixture(t)
runner, _ = countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet")
if err != nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("quiet output/error = %#v/%v", output, err)
}
}
func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{fail: true})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want execution failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" {
t.Fatalf("failed summary paths = %#v", summary)
}
if !strings.Contains(summary.Error, "prompt execution failed") {
t.Fatalf("failed summary error = %q", summary.Error)
}
assertRoutineOutputSafe(t, output)
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID)
if inspectErr != nil {
t.Fatalf("inspect %s error = %v", command, inspectErr)
}
if !strings.Contains(inspected.stdout, summary.RunID) {
t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout)
}
assertRoutineOutputSafe(t, inspected)
}
}
func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want aggregate batch failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 {
t.Fatalf("failed batch summary = %#v", summary)
}
var succeeded, failed *app.BatchReportResult
for index := range summary.Reports {
item := &summary.Reports[index]
if item.Status == summaryStatusSucceeded {
succeeded = item
} else if item.Status == summaryStatusFailed {
failed = item
}
}
if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" {
t.Fatalf("successful batch item paths = %#v", succeeded)
}
if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" {
t.Fatalf("failed batch item paths = %#v", failed)
}
if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") {
t.Fatalf("partial batch status = %q", output.stderr)
}
assertRoutineOutputSafe(t, output)
}
func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC))
second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC))
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2")
if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) {
t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err)
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID)
if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) {
t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr)
}
assertRoutineOutputSafe(t, output)
}
prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID)
if err != nil || !strings.Contains(prior.stdout, first.RunID) {
t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err)
}
}
func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runIDs := []string{
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)),
}
runID := runIDs[0]
runner, _ := countingRunner(cliExecutor{})
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath)
if err != nil {
t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err)
}
for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} {
if !strings.Contains(listed.stdout, want) {
t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout)
}
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID)
if inspectErr != nil || !strings.Contains(output.stdout, runID) {
t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr)
}
}
metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID)
if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) {
t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err)
}
}
func countingRunner(executor promptexec.Executor) (Runner, *int) {
count := new(int)
return Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
*count++
return executor, nil
},
}, count
}
func runCLICommand(runner Runner, args ...string) (commandOutput, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
err := runner.Run(context.Background(), args, &stdout, &stderr)
return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err
}
func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary {
t.Helper()
base.Clock = timeutil.FixedClock{Time: now}
output, err := runCLICommand(base, "generate", "today", "--config", configPath)
if err != nil {
t.Fatalf("generate current report: %v", err)
}
return decodeGenerateSummary(t, output.stdout)
}
func decodeGenerateSummary(t *testing.T, text string) generateSummary {
t.Helper()
var summary generateSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode generate summary: %v\n%s", err, text)
}
return summary
}
func decodeBatchSummary(t *testing.T, text string) batchSummary {
t.Helper()
var summary batchSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode batch summary: %v\n%s", err, text)
}
return summary
}
func assertRoutineOutputSafe(t *testing.T, output commandOutput) {
t.Helper()
combined := output.stdout + output.stderr
for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} {
if strings.Contains(combined, forbidden) {
t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined)
}
}
}
type cliFixture struct {
tempDir string
workspaceRoot string
configPath string
}
func newCLIFixture(t *testing.T) cliFixture {
t.Helper()
tempDir := t.TempDir()
workspaceRoot := filepath.Join(tempDir, "workspace")
server := weatherServer(t)
return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")}
}
func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) }
func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string {
t.Helper()
configPath := filepath.Join(t.TempDir(), "config.yml")
body := "weather_api:\n timezone: America/Chicago\n"
if baseURL != "" {
body += " base_url: " + baseURL + "\n"
}
body += "workspace:\n root: " + workspaceRoot + "\n"
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
return configPath
}
func weatherServer(t *testing.T) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/observations":
_, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`))
case "/conditions/current":
_, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`))
case "/forecast/hourly":
_, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`))
case "/forecast/narrative":
_, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`))
case "/alerts/active":
_, _ = w.Write([]byte(`{"data":{"alerts":[]}}`))
case "/discussion":
_, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`))
case "/weatherstories/latest":
_, _ = w.Write([]byte(`{"data":null}`))
case "/outlooks/convective":
_, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(server.Close)
return server
}
func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string {
t.Helper()
runID := "historical-" + string(reportID)
date := generatedAt.Format(timeutil.DateLayout)
dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date)
modulePath := filepath.Join(dir, "modules."+runID+".json")
dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml")
metadataPath := filepath.Join(dir, "metadata."+runID+".json")
if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil {
t.Fatalf("create historical fixture directory: %v", err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create historical metadata directory: %v", err)
}
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}})
if err != nil {
t.Fatalf("build historical module snapshot: %v", err)
}
moduleData, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal historical module snapshot: %v", err)
}
if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil {
t.Fatalf("write historical module snapshot: %v", err)
}
period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}
pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{
RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period,
}, Modules: snapshot})
if err != nil {
t.Fatalf("build historical data package: %v", err)
}
data, err := promptinput.MarshalYAML(pkg)
if err != nil {
t.Fatalf("marshal historical data package: %v", err)
}
if err := os.WriteFile(dataPath, data, 0o600); err != nil {
t.Fatalf("write historical data package: %v", err)
}
metadata := state.Metadata{
SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID),
GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive",
ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json",
}
metadataData, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("marshal historical metadata: %v", err)
}
if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil {
t.Fatalf("write historical metadata: %v", err)
}
return runID
}