660 lines
31 KiB
Go
660 lines
31 KiB
Go
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
type workflowCollector struct {
|
|
result *collect.Result
|
|
err error
|
|
calls int
|
|
}
|
|
|
|
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
|
c.calls++
|
|
if c.err != nil {
|
|
return nil, c.err
|
|
}
|
|
return c.result, nil
|
|
}
|
|
|
|
type workflowExecutor struct {
|
|
definition report.Definition
|
|
raw []byte
|
|
inspectionErr error
|
|
profile promptexec.ProfileInspection
|
|
beforePreparationErr error
|
|
afterCallbackErr error
|
|
afterPreparationErr error
|
|
validation promptexec.ValidationStatus
|
|
executeCalls int
|
|
providerCalls int
|
|
request promptexec.ExecuteRequest
|
|
beforeProvider func()
|
|
preparationDebug *promptexec.PreparationDebug
|
|
executionDebug *promptexec.ExecutionDebug
|
|
}
|
|
|
|
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
|
if e.inspectionErr != nil {
|
|
return promptexec.PromptInspection{}, e.inspectionErr
|
|
}
|
|
if id != e.definition.PromptID || version != e.definition.PromptVersion {
|
|
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
|
|
}
|
|
return validPromptInspection(e.definition), nil
|
|
}
|
|
|
|
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
profile := e.profile
|
|
if profile.ProfileID == "" {
|
|
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
|
|
}
|
|
return profile, nil
|
|
}
|
|
|
|
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
e.executeCalls++
|
|
e.request = req
|
|
if e.beforePreparationErr != nil {
|
|
return nil, e.beforePreparationErr
|
|
}
|
|
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
preparation := promptexec.Preparation{
|
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
|
|
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
|
|
}
|
|
if err := callback(preparation, e.preparationDebug); err != nil {
|
|
return nil, err
|
|
}
|
|
if e.afterCallbackErr != nil {
|
|
return nil, e.afterCallbackErr
|
|
}
|
|
if e.beforeProvider != nil {
|
|
e.beforeProvider()
|
|
}
|
|
e.providerCalls++
|
|
if e.afterPreparationErr != nil {
|
|
return nil, e.afterPreparationErr
|
|
}
|
|
validation := e.validation
|
|
if validation == "" {
|
|
validation = promptexec.ValidationPassed
|
|
}
|
|
return &promptexec.Execution{
|
|
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
|
|
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
|
|
Debug: e.executionDebug,
|
|
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
|
}, nil
|
|
}
|
|
|
|
type workflowNotifier struct {
|
|
requests []NotificationRequest
|
|
err error
|
|
}
|
|
|
|
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
|
|
n.requests = append(n.requests, req)
|
|
if n.err != nil {
|
|
return nil, n.err
|
|
}
|
|
return &NotificationResult{
|
|
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
|
|
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
|
|
}, nil
|
|
}
|
|
|
|
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
kind ReportKind
|
|
id report.ID
|
|
date time.Time
|
|
raw string
|
|
wantOutput string
|
|
}{
|
|
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
|
|
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
|
|
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
|
|
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
definition := report.DefaultRegistry().MustLookup(test.id)
|
|
bundle := workflowBundle(t)
|
|
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
|
notifier := &workflowNotifier{}
|
|
outputPath := filepath.Join(t.TempDir(), test.name+".md")
|
|
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("GenerateDetailed() error = %v", err)
|
|
}
|
|
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
|
|
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
|
|
}
|
|
if executor.request.PromptVersion != definition.PromptVersion {
|
|
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
|
|
}
|
|
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
|
|
if storeErr != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
|
|
}
|
|
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
|
|
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
|
|
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
|
|
}
|
|
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
|
|
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
|
|
}
|
|
persisted, readErr := os.ReadFile(result.DataPackagePath)
|
|
if readErr != nil {
|
|
t.Fatalf("read data package: %v", readErr)
|
|
}
|
|
if !bytes.Equal(executor.request.DataPackage, persisted) {
|
|
t.Fatal("executor data package differs from exact persisted YAML bytes")
|
|
}
|
|
managed, readErr := os.ReadFile(result.ReportPath)
|
|
if readErr != nil || !strings.Contains(string(managed), test.wantOutput) {
|
|
t.Fatalf("managed report = %q, error %v, want generated template output %q", managed, readErr, test.wantOutput)
|
|
}
|
|
copied, readErr := os.ReadFile(outputPath)
|
|
if readErr != nil || !bytes.Equal(copied, managed) || result.OutputPath != outputPath {
|
|
t.Fatalf("output copy mismatch/error/path = %v/%q", readErr, result.OutputPath)
|
|
}
|
|
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.ReportPath || notifier.requests[0].ReportPath == outputPath {
|
|
t.Fatalf("notification requests = %#v, want managed report source", notifier.requests)
|
|
}
|
|
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
|
|
if notifier.requests[0].PipelineID != wantPipeline {
|
|
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
|
|
}
|
|
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
|
|
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
|
|
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
|
|
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
|
|
}
|
|
managedName := filepath.Base(result.ReportPath)
|
|
if !strings.HasPrefix(managedName, "report.") || !strings.Contains(managedName, "_"+test.name) || !strings.HasSuffix(managedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
|
|
t.Fatalf("output names = managed %q copy %q", result.ReportPath, result.OutputPath)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type preparationFailingStore struct {
|
|
state.Store
|
|
}
|
|
|
|
type renderContextFailingStore struct {
|
|
state.Store
|
|
}
|
|
|
|
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
|
|
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
for index := range snapshot.Outputs {
|
|
snapshot.Outputs[index].Value = "invalid module value"
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
|
|
return "", errors.New("injected preparation persistence failure")
|
|
}
|
|
|
|
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
configure func(*workflowExecutor)
|
|
wantCategory promptexec.ErrorCategory
|
|
wantPreparation bool
|
|
wantExecution bool
|
|
wantRaw bool
|
|
wantProviderCall int
|
|
}{
|
|
{name: "preparation", configure: func(e *workflowExecutor) {
|
|
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
|
|
}, wantCategory: promptexec.Generation, wantPreparation: true},
|
|
{name: "credential disappears", configure: func(e *workflowExecutor) {
|
|
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
|
|
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
|
|
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
|
|
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
|
|
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "canceled", configure: func(e *workflowExecutor) {
|
|
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
|
|
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "deadline", configure: func(e *workflowExecutor) {
|
|
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
|
|
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "generation", configure: func(e *workflowExecutor) {
|
|
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
|
|
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "operational validation error", configure: func(e *workflowExecutor) {
|
|
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
|
|
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
cfg.Notify.Distributor.Enabled = false
|
|
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
|
test.configure(executor)
|
|
bundle := workflowBundle(t)
|
|
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
Collector: collector, Executor: executor,
|
|
})
|
|
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
|
|
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
|
|
}
|
|
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
|
|
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
|
}
|
|
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
|
|
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
configure func(*workflowExecutor)
|
|
wantCategory promptexec.ErrorCategory
|
|
}{
|
|
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
|
|
{name: "credential", configure: func(e *workflowExecutor) {
|
|
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
|
|
}, wantCategory: promptexec.MissingCredential},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
executor := &workflowExecutor{definition: definition}
|
|
test.configure(executor)
|
|
collector := &workflowCollector{err: errors.New("collector must not run")}
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
Collector: collector, Executor: executor,
|
|
})
|
|
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
|
|
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
|
|
}
|
|
entries, readErr := os.ReadDir(cfg.Workspace.Root)
|
|
if readErr != nil || len(entries) != 0 {
|
|
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
cfg.Notify.Distributor.Enabled = false
|
|
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
if err != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
}
|
|
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
|
bundle := workflowBundle(t)
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
|
|
})
|
|
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
|
|
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
cfg.Notify.Distributor.Enabled = false
|
|
now := workflowTime("2026-05-29T08:30:00-05:00")
|
|
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now}
|
|
resolved, err := ResolveGenerate(request, now)
|
|
if err != nil {
|
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
}
|
|
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
if err != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
}
|
|
paths, err := filesystem.Paths(resolved)
|
|
if err != nil {
|
|
t.Fatalf("Paths() error = %v", err)
|
|
}
|
|
checked := false
|
|
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
|
executor.beforeProvider = func() {
|
|
checked = true
|
|
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
|
|
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
|
|
}
|
|
}
|
|
bundle := workflowBundle(t)
|
|
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
request.Executor = executor
|
|
request.Store = filesystem
|
|
result, err := GenerateDetailed(context.Background(), request)
|
|
if err != nil || result == nil || !checked || result.OutputPath != "" {
|
|
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
raw string
|
|
configure func(*GenerateRequest, *workflowNotifier)
|
|
wantRaw bool
|
|
wantNormalized bool
|
|
wantContext bool
|
|
wantReport bool
|
|
wantOutput bool
|
|
wantNotify bool
|
|
}{
|
|
{name: "generated text decode", raw: `{`, wantRaw: true},
|
|
{name: "generated text domain", raw: `{}`, wantRaw: true},
|
|
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
req.Store = renderContextFailingStore{Store: req.Store}
|
|
}, wantRaw: true, wantNormalized: true},
|
|
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
|
|
}, wantRaw: true, wantNormalized: true},
|
|
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
blocker := filepath.Join(t.TempDir(), "report-blocker")
|
|
if err := os.Mkdir(blocker, 0o700); err != nil {
|
|
t.Fatalf("create report blocker: %v", err)
|
|
}
|
|
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
|
|
}, wantRaw: true, wantNormalized: true, wantContext: true},
|
|
{name: "output copy", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
req.OutputPath = t.TempDir()
|
|
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true},
|
|
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
|
|
notifier.err = errors.New("notification rejected")
|
|
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true, wantNotify: true},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
|
notifier := &workflowNotifier{}
|
|
bundle := workflowBundle(t)
|
|
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
if err != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
}
|
|
req := GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
|
Executor: executor, Notifier: notifier, Store: filesystem,
|
|
}
|
|
if test.configure != nil {
|
|
test.configure(&req, notifier)
|
|
}
|
|
result, err := GenerateDetailed(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
|
|
}
|
|
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
|
|
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
|
|
(result.OutputPath != "") != test.wantOutput || (result.NotificationPath != "") != test.wantNotify {
|
|
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q notification %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath, result.NotificationPath)
|
|
}
|
|
if test.wantRaw {
|
|
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
|
|
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
|
|
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
|
|
}
|
|
}
|
|
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.ReportPath) {
|
|
t.Fatalf("notification requests = %#v", notifier.requests)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
createCollision func(string, report.Resolved) error
|
|
wantProviderCalls int
|
|
wantPreparationFile bool
|
|
}{
|
|
{
|
|
name: "preparation debug",
|
|
createCollision: func(root string, resolved report.Resolved) error {
|
|
path := workflowDebugRunPath(root, resolved)
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, []byte("not a directory"), 0o600)
|
|
},
|
|
},
|
|
{
|
|
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
|
|
createCollision: func(root string, resolved report.Resolved) error {
|
|
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
|
|
},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
cfg.Notify.Distributor.Enabled = false
|
|
now := workflowTime("2026-05-29T08:30:00-05:00")
|
|
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now}
|
|
resolved, err := ResolveGenerate(request, now)
|
|
if err != nil {
|
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
}
|
|
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
|
|
if err := test.createCollision(debugRoot, resolved); err != nil {
|
|
t.Fatalf("create debug collision: %v", err)
|
|
}
|
|
bundle := workflowBundle(t)
|
|
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
|
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
request.Executor = executor
|
|
request.LLMDebugDir = debugRoot
|
|
result, err := GenerateDetailed(context.Background(), request)
|
|
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
|
|
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
|
|
}
|
|
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
|
|
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
|
}
|
|
if test.wantPreparationFile && result.MetadataPath == "" {
|
|
t.Fatal("execution debug failure lost previously persisted metadata")
|
|
}
|
|
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
|
|
_, statErr := os.Stat(preparationDebug)
|
|
if (statErr == nil) != test.wantPreparationFile {
|
|
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func workflowDebugRunPath(root string, resolved report.Resolved) string {
|
|
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
|
|
}
|
|
|
|
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
|
|
switch id {
|
|
case report.Daily:
|
|
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
|
|
case report.Today:
|
|
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
|
|
case report.Tomorrow:
|
|
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
|
|
case report.Hourly:
|
|
return []string{"hourly/index.md"}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedSelectsPriorSnapshotsForRetainedReports(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
kind ReportKind
|
|
id report.ID
|
|
date time.Time
|
|
raw string
|
|
wantPrior bool
|
|
wantRecentChanges bool
|
|
}{
|
|
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
|
|
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
|
|
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantPrior: true, wantRecentChanges: true},
|
|
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON()},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
cfg := workflowConfig(t)
|
|
cfg.Notify.Distributor.Enabled = false
|
|
definition := report.DefaultRegistry().MustLookup(test.id)
|
|
firstBundle := workflowBundle(t)
|
|
setWorkflowTemperatures(&firstBundle, 45)
|
|
first, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:00:00-05:00"),
|
|
Collector: &workflowCollector{result: &collect.Result{Bundle: &firstBundle}},
|
|
Executor: &workflowExecutor{definition: definition, raw: []byte(test.raw)},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("first GenerateDetailed() error = %v", err)
|
|
}
|
|
secondBundle := workflowBundle(t)
|
|
setWorkflowTemperatures(&secondBundle, 85)
|
|
second, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"),
|
|
Collector: &workflowCollector{result: &collect.Result{Bundle: &secondBundle}},
|
|
Executor: &workflowExecutor{definition: definition, raw: []byte(test.raw)},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("second GenerateDetailed() error = %v", err)
|
|
}
|
|
if test.wantPrior && (second.PriorSnapshot == nil || second.PriorSnapshot.Metadata.RunID != first.Metadata.RunID || second.PriorSnapshot.Metadata.ReportID != test.id) {
|
|
t.Fatalf("prior snapshot = %#v, want first %s run %q", second.PriorSnapshot, test.id, first.Metadata.RunID)
|
|
}
|
|
if !test.wantPrior && second.PriorSnapshot != nil {
|
|
t.Fatalf("prior snapshot = %#v, want none for non-overlapping rolling window", second.PriorSnapshot)
|
|
}
|
|
if (len(second.RecentChanges) > 0) != test.wantRecentChanges {
|
|
t.Fatalf("recent changes = %#v, want present %t", second.RecentChanges, test.wantRecentChanges)
|
|
}
|
|
if (len(second.DataPackage.RecentChanges.Items) > 0) != test.wantRecentChanges {
|
|
t.Fatalf("data package recent changes = %#v, want present %t", second.DataPackage.RecentChanges.Items, test.wantRecentChanges)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func setWorkflowTemperatures(bundle *weatherdata.Bundle, temperature float64) {
|
|
for index := range bundle.Hourly.Periods {
|
|
value := temperature
|
|
bundle.Hourly.Periods[index].TemperatureF = &value
|
|
}
|
|
}
|
|
|
|
func workflowConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
cfg := config.Defaults()
|
|
cfg.Workspace.Root = t.TempDir()
|
|
cfg.WeatherAPI.Timezone = "America/Chicago"
|
|
cfg.Location.ID = "home"
|
|
cfg.Location.Name = "Testville"
|
|
cfg.Location.Region = "MO"
|
|
cfg.Notify.Distributor.Enabled = true
|
|
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
|
|
return cfg
|
|
}
|
|
|
|
func workflowBundle(t *testing.T) weatherdata.Bundle {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
if err != nil {
|
|
t.Fatalf("read bundle fixture: %v", err)
|
|
}
|
|
var bundle weatherdata.Bundle
|
|
if err := json.Unmarshal(data, &bundle); err != nil {
|
|
t.Fatalf("decode bundle fixture: %v", err)
|
|
}
|
|
future := bundle.Hourly.Periods[0]
|
|
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
|
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
|
|
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
|
|
futureNarrative := bundle.Narrative.Periods[0]
|
|
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
|
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
|
|
futureNarrative.Name = "Tomorrow"
|
|
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
|
|
return bundle
|
|
}
|
|
|
|
func workflowTime(value string) time.Time {
|
|
parsed, err := time.Parse(time.RFC3339, value)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func validHourlyWorkflowJSON() string {
|
|
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`
|
|
}
|
|
|
|
func validTomorrowWorkflowJSON() string {
|
|
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
|
}
|
|
|
|
func validTodayWorkflowJSON() string {
|
|
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
|
}
|
|
|
|
func validDailyWorkflowJSON() string {
|
|
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
|
|
}
|