Run report generation without workspace state

This commit is contained in:
2026-08-01 19:52:22 +00:00
parent 4bdba6f2b7
commit 7ffc3dc603
21 changed files with 343 additions and 3464 deletions

View File

@@ -124,9 +124,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
PromptID: request.PromptID, PromptID: request.PromptID,
PromptVersion: request.PromptVersion, PromptVersion: request.PromptVersion,
ProfileID: request.ProfileID, ProfileID: request.ProfileID,
Inputs: map[string]promptkit.ArtifactRef{ Inputs: map[string]promptkit.ArtifactRef{"data_package": promptkit.Inline(string(append([]byte(nil), request.DataPackage...)))},
"data_package": promptkit.InlineWithURI(request.DataPackagePath, string(append([]byte(nil), request.DataPackage...))),
},
}) })
if err != nil { if err != nil {
return nil, classifyError(err) return nil, classifyError(err)
@@ -134,7 +132,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
defer prepared.Discard() defer prepared.Discard()
details := prepared.Details() details := prepared.Details()
preparation, debug := preparationValues(details, request.DataPackagePath, request.CaptureDebug) preparation, debug := preparationValues(details, request.CaptureDebug)
if preparedCallback != nil { if preparedCallback != nil {
if err := preparedCallback(preparation, debug); err != nil { if err := preparedCallback(preparation, debug); err != nil {
return nil, err return nil, err
@@ -145,7 +143,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
if err != nil { if err != nil {
return nil, classifyError(err) return nil, classifyError(err)
} }
return executionValue(result, request.DataPackagePath, request.CaptureDebug), nil return executionValue(result, request.CaptureDebug), nil
} }
func outputContract(value promptkit.OutputContract) promptexec.OutputContract { func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
@@ -156,7 +154,7 @@ func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
} }
} }
func preparationValues(value promptkit.PreparedRun, dataPackagePath string, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) { func preparationValues(value promptkit.PreparedRun, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
preparation := promptexec.Preparation{ preparation := promptexec.Preparation{
PromptID: value.PromptID, PromptID: value.PromptID,
PromptVersion: value.PromptVersion, PromptVersion: value.PromptVersion,
@@ -170,7 +168,6 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
StartedAt: value.StartTime, StartedAt: value.StartTime,
EndedAt: value.EndTime, EndedAt: value.EndTime,
Duration: time.Duration(value.DurationMS) * time.Millisecond, Duration: time.Duration(value.DurationMS) * time.Millisecond,
DataPackagePath: dataPackagePath,
} }
if !captureDebug { if !captureDebug {
return preparation, nil return preparation, nil
@@ -186,7 +183,7 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
return preparation, debug return preparation, debug
} }
func executionValue(value *promptkit.RunResult, dataPackagePath string, captureDebug bool) *promptexec.Execution { func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.Execution {
if value == nil { if value == nil {
return nil return nil
} }
@@ -218,7 +215,6 @@ func executionValue(value *promptkit.RunResult, dataPackagePath string, captureD
EndedAt: value.EndTime, EndedAt: value.EndTime,
Duration: value.Duration, Duration: value.Duration,
Validation: validation, Validation: validation,
DataPackagePath: dataPackagePath,
RawOutput: []byte(value.RawOutput), RawOutput: []byte(value.RawOutput),
} }
if captureDebug { if captureDebug {

View File

@@ -210,7 +210,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
callbackCalls := 0 callbackCalls := 0
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
callbackCalls++ callbackCalls++
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.DataPackagePath != request.DataPackagePath || preparation.ModelName != "test-model" { if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" {
t.Fatalf("preparation = %#v", preparation) t.Fatalf("preparation = %#v", preparation)
} }
if debug != nil { if debug != nil {
@@ -227,7 +227,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
if callbackCalls != 1 || client.callCount() != 1 { if callbackCalls != 1 || client.callCount() != 1 {
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount()) t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
} }
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content || result.DataPackagePath != request.DataPackagePath { if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content {
t.Fatalf("result = %#v", result) t.Fatalf("result = %#v", result)
} }
if result.Debug != nil { if result.Debug != nil {
@@ -254,7 +254,6 @@ func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
PromptVersion: "2.0.0", PromptVersion: "2.0.0",
ProfileID: "weather-light", ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"), DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
DataPackagePath: "data-packages/hourly/data_package.yaml",
} }
var preparation promptexec.Preparation var preparation promptexec.Preparation
prepared := false prepared := false
@@ -288,7 +287,7 @@ func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
if _, err := adapter.Execute(context.Background(), request, nil); err != nil { if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
t.Fatalf("Execute() error = %v", err) t.Fatalf("Execute() error = %v", err)
} }
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != request.DataPackagePath || reader.ref.Body != string(request.DataPackage) { if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) {
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref) t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
} }
} }
@@ -531,7 +530,6 @@ func testExecuteRequest() promptexec.ExecuteRequest {
PromptVersion: "2.0.0", PromptVersion: "2.0.0",
ProfileID: "test-profile", ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"), DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
DataPackagePath: "data-packages/daily/data_package.yaml",
} }
} }

View File

@@ -21,7 +21,6 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
) )
@@ -53,7 +52,6 @@ type GenerateRequest struct {
Collector Collector Collector Collector
Notifier Notifier Notifier Notifier
Executor promptexec.Executor Executor promptexec.Executor
Store state.Store
} }
type BatchRequest struct { type BatchRequest struct {
@@ -65,7 +63,6 @@ type BatchRequest struct {
LLMDebugDir string LLMDebugDir string
Collector Collector Collector Collector
Executor promptexec.Executor Executor promptexec.Executor
Store state.Store
Notifier Notifier Notifier Notifier
} }
@@ -85,20 +82,21 @@ type ReportFacts struct {
} }
type ReportResult struct { type ReportResult struct {
ModuleSnapshot module.Snapshot ReportID report.ID
ModuleSnapshotPath string ReportName string
DataPackage promptinput.Package PromptID string
DataPackagePath string PromptVersion string
PreparationPath string RunID string
ExecutionPath string GeneratedAt time.Time
Timezone string
ValidPeriod timeutil.Period
ProfileID string
BackendID string
ModelName string
SourceWarnings []weatherdata.SourceWarning
ValidationStatus promptexec.ValidationStatus
LLMDebugPath string LLMDebugPath string
ReportPath string
OutputPath string OutputPath string
Metadata state.Metadata
MetadataPath string
GeneratedTextRawPath string
GeneratedTextPath string
RenderContextPath string
Notification *NotificationResult Notification *NotificationResult
} }
@@ -144,13 +142,14 @@ type BatchReportResult struct {
NotificationError string `json:"notificationError,omitempty"` NotificationError string `json:"notificationError,omitempty"`
GeneratedAt time.Time `json:"generatedAt"` GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"` ValidPeriod timeutil.Period `json:"validPeriod"`
DataPackagePath string `json:"dataPackagePath,omitempty"` Timezone string `json:"timezone"`
PreparationPath string `json:"preparationPath,omitempty"` ProfileID string `json:"profileId,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"` BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"` LLMDebugPath string `json:"llmDebugPath,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"` OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
} }
type BatchError struct { type BatchError struct {
@@ -260,14 +259,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
if err != nil { if err != nil {
return nil, err return nil, err
} }
result := initialReportResult(req, resolved, PromptInspectionResult{})
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved) outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
if err != nil { if err != nil {
return nil, err return result, err
} }
req.OutputPath = outputPath req.OutputPath = outputPath
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir) debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil { if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err) return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
} }
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{ inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
Resolved: resolved, Resolved: resolved,
@@ -275,11 +275,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Promptkit: req.Config.Promptkit, Promptkit: req.Config.Promptkit,
}) })
if err != nil { if err != nil {
return nil, err return result, err
} }
result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName
collection, err := collectWeather(ctx, req.Config, req.Collector) collection, err := collectWeather(ctx, req.Config, req.Collector)
if err != nil { if err != nil {
return nil, err return result, err
} }
return generatePromptReport(ctx, promptReportRequest{ return generatePromptReport(ctx, promptReportRequest{
GenerateRequest: req, GenerateRequest: req,
@@ -287,6 +288,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Collection: *collection, Collection: *collection,
Inspection: inspection, Inspection: inspection,
DebugWriter: debugWriter, DebugWriter: debugWriter,
Result: result,
}) })
} }
@@ -339,14 +341,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
return nil, err return nil, err
} }
if req.Batch == BatchEvening || req.Batch == BatchMorning { if req.Batch == BatchEvening || req.Batch == BatchMorning {
store := req.Store
if store == nil {
defaultStore, err := defaultStore(req.Config)
if err != nil {
return nil, err
}
store = defaultStore
}
startedAt := now startedAt := now
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt} result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
for _, planned := range plannedReports { for _, planned := range plannedReports {
@@ -362,7 +356,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
OutputPath: outputPath, OutputPath: outputPath,
Notifier: req.Notifier, Notifier: req.Notifier,
Executor: req.Executor, Executor: req.Executor,
Store: store,
}, },
Resolved: resolved, Resolved: resolved,
Collection: *collection, Collection: *collection,
@@ -398,13 +391,14 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
} }
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) { func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
item.DataPackagePath = result.DataPackagePath
item.PreparationPath = result.PreparationPath
item.ExecutionPath = result.ExecutionPath
item.LLMDebugPath = result.LLMDebugPath item.LLMDebugPath = result.LLMDebugPath
item.ReportPath = result.ReportPath
item.OutputPath = result.OutputPath item.OutputPath = result.OutputPath
item.MetadataPath = result.MetadataPath item.ProfileID = result.ProfileID
item.BackendID = result.BackendID
item.ModelName = result.ModelName
item.Timezone = result.Timezone
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
item.ValidationStatus = result.ValidationStatus
if result.Notification != nil { if result.Notification != nil {
item.NotificationStatus = result.Notification.Status item.NotificationStatus = result.Notification.Status
item.NotificationRunID = result.Notification.RunID item.NotificationRunID = result.Notification.RunID
@@ -451,6 +445,7 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
RunID: metadata.RunID, RunID: metadata.RunID,
GeneratedAt: metadata.GeneratedAt, GeneratedAt: metadata.GeneratedAt,
ValidPeriod: metadata.ValidPeriod, ValidPeriod: metadata.ValidPeriod,
Timezone: "",
} }
} }
@@ -612,78 +607,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda
return bundle, nil return bundle, nil
} }
type finalizeRenderedReportRequest struct {
Config config.Config
Store state.Store
Resolved report.Resolved
Metadata state.Metadata
MetadataPath string
ExecutionArtifact *state.PromptExecutionArtifact
RenderedReportPath string
OutputPath string
Notifier Notifier
GenerationErr error
noNotify bool
}
type finalizeRenderedReportResult struct {
OutputPath string
Metadata state.Metadata
MetadataPath string
Notification *NotificationResult
}
func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) {
if req.Store == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("state store is required")
}
if req.RenderedReportPath == "" {
return finalizeRenderedReportResult{}, fmt.Errorf("rendered report path is required for report %q", req.Resolved.Definition.ID)
}
if req.ExecutionArtifact == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
}
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
if req.OutputPath != "" && req.GenerationErr == nil {
if req.OutputPath != req.RenderedReportPath {
if err := fileutil.CopyFileAtomic(req.RenderedReportPath, req.OutputPath); err != nil {
return result, err
}
result.OutputPath = req.OutputPath
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
paths.OutputPath = req.OutputPath
}); err != nil {
return result, err
}
} else {
result.OutputPath = req.OutputPath
}
}
metadata := req.Metadata
metadata.RenderedReportPath = req.RenderedReportPath
metadataPath, err := req.Store.SaveMetadata(ctx, metadata)
if err != nil {
return result, err
}
result.Metadata = metadata
result.MetadataPath = metadataPath
if req.GenerationErr != nil {
return result, req.GenerationErr
}
if req.noNotify {
return result, nil
}
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.OutputPath, metadata.RunID, metadata.GeneratedAt, req.Notifier)
result.Notification = notification
if err != nil {
return result, err
}
return result, nil
}
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) { func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
notifier, enabled := reportNotifier(cfg, notifier) notifier, enabled := reportNotifier(cfg, notifier)
if !enabled { if !enabled {
@@ -965,7 +888,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected
} }
} }
func promptMetadata(metadata state.Metadata) promptinput.Metadata { func promptMetadata(metadata briefing.Metadata) promptinput.Metadata {
return promptinput.Metadata{ return promptinput.Metadata{
RunID: metadata.RunID, RunID: metadata.RunID,
ReportID: metadata.ReportID, ReportID: metadata.ReportID,
@@ -1008,10 +931,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext {
return &location return &location
} }
func defaultStore(cfg config.Config) (*state.FilesystemStore, error) {
return state.NewFilesystemStore(cfg.Workspace)
}
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error { func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
if err == nil { if err == nil {
return nil return nil

View File

@@ -1,85 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now string
wantPrompts int
}{
{name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3},
{name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
now := mustParse(test.now)
req := BatchRequest{Config: cfg, Batch: test.batch, Now: now, WorkingDir: t.TempDir()}
candidates, err := batchInspectionCandidates(req, now)
if err != nil {
t.Fatalf("batchInspectionCandidates() error = %v", err)
}
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
"weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
}, prompts: map[string]promptexec.PromptInspection{}}
for _, candidate := range candidates {
executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition)
}
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
return nil, errors.New("collection reached")
})
req.Executor = executor
req.Collector = collector
_, err = RunBatchDetailed(context.Background(), req)
if err == nil || err.Error() != "collection reached" {
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
}
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
}
})
}
}
func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) {
item := BatchReportResult{}
copyBatchReportPaths(&item, &ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
})
data, err := json.Marshal(item)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("batch item omits reached paths:\n%s", text)
}
}
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
return f(ctx, req)
}
var _ Collector = collectorFunc(nil)

View File

@@ -1,449 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type assembledBatchExecutor struct {
definitions map[string]report.Definition
promptRequests []string
profileRequests []string
executeRequests []promptexec.ExecuteRequest
failures map[int]error
active int
maxActive int
}
func newAssembledBatchExecutor() *assembledBatchExecutor {
definitions := make(map[string]report.Definition)
for _, definition := range report.DefaultRegistry().All() {
definitions[definition.PromptID] = definition
}
return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)}
}
func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
e.promptRequests = append(e.promptRequests, id+"@"+version)
definition, ok := e.definitions[id]
if !ok || definition.PromptVersion != version {
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
}
return logicalPromptInspection(definition), nil
}
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
e.profileRequests = append(e.profileRequests, id)
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
call := len(e.executeRequests)
e.executeRequests = append(e.executeRequests, req)
e.active++
if e.active > e.maxActive {
e.maxActive = e.active
}
defer func() { e.active-- }()
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if err := e.failures[call]; err != nil {
return nil, err
}
definition := e.definitions[req.PromptID]
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(generatedTextForPrompt(req.PromptID)),
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
type assembledBatchNotifier struct {
reportRequests []NotificationRequest
batchRequests []batchNotificationRequest
batchResult *NotificationResult
batchErr error
}
func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.reportRequests = append(n.reportRequests, req)
return nil, errors.New("per-report notification must be suppressed")
}
func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) {
n.batchRequests = append(n.batchRequests, req)
if n.batchErr != nil {
return nil, n.batchErr
}
if n.batchResult != nil {
result := *n.batchResult
if result.PipelineID == "" {
result.PipelineID = req.PipelineID
}
if result.BundleID == "" {
result.BundleID = req.BundleID
}
if result.IdempotencyKey == "" {
result.IdempotencyKey = req.IdempotencyKey
}
return &result, nil
}
return &NotificationResult{
RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now time.Time
wantIDs []report.ID
wantCopies []string
}{
{name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}},
{name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: test.batch, Now: test.now, WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: collector, Executor: executor,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 {
t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed)
}
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
}
if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests)
}
for index, item := range result.Reports {
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {
t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index])
}
if executor.executeRequests[index].PromptID != item.PromptID {
t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" {
t.Fatalf("successful report paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
if filepath.Base(item.OutputPath) != test.wantCopies[index] {
t.Fatalf("output = %q, want %q", item.OutputPath, test.wantCopies[index])
}
assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath)
renderedReport, readErr := os.ReadFile(item.ReportPath)
if readErr != nil {
t.Fatalf("read rendered report: %v", readErr)
}
copied, readErr := os.ReadFile(item.OutputPath)
if readErr != nil || !bytes.Equal(renderedReport, copied) {
t.Fatalf("output mismatch/error = %v", readErr)
}
}
})
}
}
func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(),
OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"),
Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed)
}
if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" {
t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification)
}
for index, item := range result.Reports {
if index == 1 {
if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) {
t.Fatalf("failed item = %#v", item)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" {
t.Fatalf("failed item reached paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
continue
}
if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" {
t.Fatalf("continued item %d = %#v", index, item)
}
}
}
func TestRunBatchDetailedNotificationLifecycle(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("batch disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
cfg.Notify.Distributor.Batch.Enabled = false
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("all success", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted",
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
outputDir := filepath.Join(t.TempDir(), "output")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 {
t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests))
}
outputPaths := make(map[string]struct{}, len(result.Reports))
for _, item := range result.Reports {
outputPaths[item.OutputPath] = struct{}{}
}
request := notifier.batchRequests[0]
if len(request.IncludedReports) != len(result.Reports) {
t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports))
}
for _, file := range request.Files {
if _, ok := outputPaths[file.SourcePath]; !ok || !strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" {
t.Fatalf("notification file = %#v, want selected Markdown output source", file)
}
}
})
t.Run("upload failure", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" {
t.Fatalf("result = %#v, want successful reports and failed notification", result)
}
for _, item := range result.Reports {
if item.Status != "succeeded" {
t.Fatalf("report item = %#v, want success despite notification failure", item)
}
}
if !strings.Contains(result.Notification.Error, "batch upload rejected") {
t.Fatalf("notification error = %#v", result.Notification)
}
})
t.Run("status report", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted",
StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Failed != 0 || result.Notification == nil {
t.Fatalf("result/error = %#v/%v", result, err)
}
if result.Notification.Status != "accepted" || result.Notification.RunID != "batch-notification-run" {
t.Fatalf("notification = %#v", result.Notification)
}
})
}
func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01")
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(),
})
if err != nil || result.Failed != 0 || len(result.Reports) != 3 {
t.Fatalf("result/error = %#v/%v", result, err)
}
seenRuns := make(map[string]struct{})
seenDebug := make(map[string]struct{})
dailyDates := make(map[string]BatchReportResult)
for _, item := range result.Reports {
if _, exists := seenRuns[item.RunID]; exists {
t.Fatalf("duplicate run ID %q", item.RunID)
}
seenRuns[item.RunID] = struct{}{}
if _, exists := seenDebug[item.LLMDebugPath]; exists {
t.Fatalf("duplicate debug path %q", item.LLMDebugPath)
}
seenDebug[item.LLMDebugPath] = struct{}{}
if item.ReportID == report.Daily {
date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02")
dailyDates[date] = item
}
}
for _, date := range []string{"2026-05-31", "2026-06-01"} {
item, ok := dailyDates[date]
if !ok {
t.Fatalf("daily items = %#v, want %s", dailyDates, date)
}
if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") {
t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath)
}
wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID)
if item.LLMDebugPath != wantDebugPrefix {
t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix)
}
assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json"))
}
}
func assembledBatchConfig(t *testing.T, notify bool) config.Config {
t.Helper()
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = notify
cfg.Notify.Distributor.Batch.Enabled = notify
return cfg
}
func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle {
t.Helper()
bundle := workflowBundle(t)
location := mustLoadTestLocation(t, "America/Chicago")
for _, date := range dates {
periods := fullDayPeriods(t, date, location)
for index := range periods {
temperature := float64(60 + index)
periods[index].TemperatureF = &temperature
periods[index].TextDescription = "Partly cloudy"
}
bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...)
}
return bundle
}
func generatedTextForPrompt(promptID string) string {
switch promptID {
case "weather.today_generated_text":
return validTodayWorkflowJSON()
case "weather.tomorrow_generated_text":
return validTomorrowWorkflowJSON()
case "weather.daily_generated_text":
return validDailyWorkflowJSON()
case "weather.hourly_generated_text":
return validHourlyWorkflowJSON()
default:
return ""
}
}
func assertBatchPathsExist(t *testing.T, paths ...string) {
t.Helper()
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected path %q: %v", path, err)
}
}
}
func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) {
t.Helper()
data, err := os.ReadFile(item.MetadataPath)
if err != nil {
t.Fatalf("read metadata %q: %v", item.MetadataPath, err)
}
var metadata state.Metadata
if err := json.Unmarshal(data, &metadata); err != nil {
t.Fatalf("decode metadata %q: %v", item.MetadataPath, err)
}
if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID ||
item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath ||
item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath {
t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata)
}
}
func loadBatchDataPackage(t *testing.T, path string) promptinput.Package {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read data package: %v", err)
}
pkg, err := promptinput.LoadYAML(data)
if err != nil {
t.Fatalf("LoadYAML() error = %v", err)
}
return pkg
}

View File

@@ -0,0 +1,103 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type generationCollector struct {
bundle *weatherdata.Bundle
err error
}
func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
return &collect.Result{Bundle: c.bundle}, c.err
}
type generationExecutor struct{ called bool }
func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
definition := report.DefaultRegistry().MustLookup(report.Daily)
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
}
func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
return nil, err
}
e.called = true
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil
}
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
bundle := generationBundle(t)
executor := &generationExecutor{}
workingDir := t.TempDir()
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
t.Fatalf("result = %#v", result)
}
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
t.Fatalf("workspace state = %v, want absent", err)
}
data, err := os.ReadFile(result.OutputPath)
if err != nil || len(data) == 0 {
t.Fatalf("output = %q, error = %v", data, err)
}
}
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
collectionErr := errors.New("weather source unavailable")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{},
})
if !errors.Is(err, collectionErr) {
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
}
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
t.Fatalf("result = %#v", result)
}
}
func generationBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
return bundle
}
func generationTime(value string) time.Time {
parsed, _ := time.Parse(time.RFC3339, value)
return parsed
}
var _ promptexec.Executor = (*generationExecutor)(nil)
var _ Collector = generationCollector{}
var _ = report.Daily

View File

@@ -33,7 +33,7 @@ type SourceInspection struct {
} }
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) { func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
store, err := defaultStore(req.Config) store, err := state.NewFilesystemStore(req.Config.Workspace)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -94,7 +94,7 @@ type runInspection struct {
} }
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) { func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
store, err := defaultStore(req.Config) store, err := state.NewFilesystemStore(req.Config.Workspace)
if err != nil { if err != nil {
return runInspection{}, err return runInspection{}, err
} }

View File

@@ -1,129 +0,0 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func TestResolveReportOutputPath(t *testing.T) {
workingDir := t.TempDir()
cfg := config.Defaults()
now := workflowTime("2026-05-29T08:30:00-05:00")
for _, test := range []struct {
report ReportKind
date string
name string
}{
{report: ReportDaily, date: "2026-05-30T12:00:00-05:00", name: "daily-2026-05-30.md"},
{report: ReportToday, date: "2026-05-29T12:00:00-05:00", name: "today.md"},
{report: ReportTomorrow, name: "tomorrow.md"},
{report: ReportHourly, name: "hourly.md"},
} {
t.Run(string(test.report), func(t *testing.T) {
req := GenerateRequest{Config: cfg, Report: test.report}
if test.date != "" {
req.Date = workflowTime(test.date)
}
resolved, err := ResolveGenerate(req, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
path, err := resolveReportOutputPath(workingDir, "", resolved)
if err != nil {
t.Fatalf("resolveReportOutputPath() error = %v", err)
}
if path != filepath.Join(workingDir, test.name) {
t.Fatalf("path = %q, want %q", path, filepath.Join(workingDir, test.name))
}
})
}
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-30T12:00:00-05:00"),
}, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
absoluteDir := t.TempDir()
for _, test := range []struct {
override string
want string
}{
{override: filepath.Join("reports", "custom.md"), want: filepath.Join(workingDir, "reports", "custom.md")},
{override: filepath.Join(absoluteDir, "custom.md"), want: filepath.Join(absoluteDir, "custom.md")},
} {
path, err := resolveReportOutputPath(workingDir, test.override, resolved)
if err != nil {
t.Fatalf("resolveReportOutputPath(%q) error = %v", test.override, err)
}
if path != filepath.Clean(test.want) {
t.Fatalf("path = %q, want %q", path, filepath.Clean(test.want))
}
}
}
func TestGenerateDetailedRejectsInvalidOutputBeforeCollection(t *testing.T) {
cfg := workflowConfig(t)
collector := &workflowCollector{err: errors.New("collection must not run")}
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: t.TempDir(),
Collector: collector,
})
if err == nil || collector.calls != 0 {
t.Fatalf("GenerateDetailed() error/calls = %v/%d, want invalid output before collection", err, collector.calls)
}
}
func TestGenerateDetailedPreservesExistingOutputWhenGenerationFails(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
workingDir := t.TempDir()
outputPath := filepath.Join(workingDir, "daily-2026-05-29.md")
if err := os.WriteFile(outputPath, []byte("existing report"), 0o600); err != nil {
t.Fatalf("write existing output: %v", err)
}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: &workflowExecutor{definition: report.DefaultRegistry().MustLookup(report.Daily), raw: []byte(`{}`)},
})
if err == nil || result == nil || result.OutputPath != "" {
t.Fatalf("result/error/output = %#v/%v/%q, want failed generation without output publication", result, err, result.OutputPath)
}
data, readErr := os.ReadFile(outputPath)
if readErr != nil || string(data) != "existing report" {
t.Fatalf("output after failure = %q, error %v, want preserved content", data, readErr)
}
}
func TestRunBatchDetailedUsesWorkingDirectoryForOutput(t *testing.T) {
cfg := assembledBatchConfig(t, false)
workingDir := t.TempDir()
bundle := assembledBatchBundle(t, "2026-05-31")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: workingDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: newAssembledBatchExecutor(),
})
if err != nil || result == nil || result.Failed != 0 {
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
}
if len(result.Reports) != 2 {
t.Fatalf("reports = %#v, want Tomorrow and Daily", result.Reports)
}
if result.Reports[0].OutputPath != filepath.Join(workingDir, "tomorrow.md") ||
result.Reports[1].OutputPath != filepath.Join(workingDir, "daily-2026-05-31.md") {
t.Fatalf("output paths = %#v, want working-directory defaults", result.Reports)
}
}

View File

@@ -1,523 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failGeneratedText = "generated text"
failRenderContext = "render context"
failRenderedReportPath = "rendered report path"
)
type failingPersistenceStore struct {
state.Store
failOperation string
failExecutionCall int
failMetadataCall int
executionCalls int
metadataCalls int
renderedReportPath string
}
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
s.executionCalls++
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
return "", errors.New("injected prompt execution persistence failure")
}
return s.Store.SavePromptExecution(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
if s.failOperation == failGeneratedText {
return "", errors.New("injected generated text persistence failure")
}
return s.Store.SaveGeneratedText(ctx, resolved, data)
}
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
if s.failOperation == failRenderContext {
return "", errors.New("injected render context persistence failure")
}
return s.Store.SaveRenderContext(ctx, resolved, value)
}
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
if s.failOperation == failRenderedReportPath {
return s.renderedReportPath, nil
}
return s.Store.PrepareRenderedReport(ctx, resolved)
}
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
s.metadataCalls++
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
return "", errors.New("injected metadata persistence failure")
}
return s.Store.SaveMetadata(ctx, metadata)
}
type artifactPathExecutor struct {
beforePreparationErr error
afterPreparationErr error
validation promptexec.ValidationStatus
}
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
}, nil); err != nil {
return nil, err
}
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`),
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
}, nil
}
type successfulNotifier struct{}
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
}
type failingNotifier struct{}
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return nil, errors.New("injected notification failure")
}
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
tests := []struct {
name string
failOperation string
failMetadataCall int
output bool
notify bool
want reachedPromptArtifacts
}{
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
{name: "rendered report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
{name: "output then metadata", failOperation: failMetadata, failMetadataCall: 5, output: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
req.Store = store
if test.output {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
})
}
}
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
tests := []struct {
name string
executor artifactPathExecutor
want reachedPromptArtifacts
wantExecutionStatus state.PromptExecutionStatus
wantExecutionPaths state.PromptExecutionPaths
wantRawExecution bool
}{
{
name: "preparation failure",
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
want: reachedPromptArtifacts{preparation: true, metadata: true},
},
{
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
wantExecutionStatus: state.PromptExecutionFailed,
},
{
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
wantExecutionStatus: state.PromptExecutionValidationRejected,
wantRawExecution: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, test.executor)
if test.wantRawExecution {
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
if test.wantExecutionStatus != "" {
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if loadErr != nil {
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
}
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
}
}
})
}
}
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
result, err := generatePromptReport(context.Background(), req)
if err != nil {
t.Fatalf("generatePromptReport() error = %v", err)
}
want := state.PromptExecutionPaths{
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
OutputPath: paths.output,
}
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
data, err := os.ReadFile(result.ExecutionPath)
if err != nil {
t.Fatalf("read execution artifact: %v", err)
}
text := string(data)
for _, forbidden := range []string{
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
"credential", "secret-value",
} {
if strings.Contains(text, forbidden) {
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
}
}
}
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
tests := []struct {
name string
failOperation string
failExecutionCall int
failMetadataCall int
requestOutput bool
failOutput bool
notify bool
notificationFailure bool
wantExecution reachedExecutionArtifacts
wantResult reachedPromptArtifacts
}{
{
name: "normalized text write", failOperation: failGeneratedText,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
},
{
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context write", failOperation: failRenderContext,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "rendered report write", failOperation: failRenderedReportPath,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "rendered report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output write", requestOutput: true, failOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "output metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{
Store: req.Store, failOperation: test.failOperation,
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
}
if test.failOperation == failRenderedReportPath {
store.renderedReportPath = t.TempDir()
}
req.Store = store
if test.requestOutput {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.failOutput {
blocker := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
t.Fatalf("write output blocker: %v", err)
}
req.OutputPath = filepath.Join(blocker, "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
if test.notificationFailure {
req.Notifier = failingNotifier{}
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
})
}
}
type reachedExecutionArtifacts struct {
raw bool
normalized bool
renderContext bool
report bool
output bool
}
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
result := state.PromptExecutionPaths{}
if reached.raw {
result.RawOutputPath = paths.GeneratedTextRaw
}
if reached.normalized {
result.GeneratedTextPath = paths.GeneratedText
}
if reached.renderContext {
result.RenderContextPath = paths.RenderContext
}
if reached.report {
result.RenderedReportPath = paths.RenderedReport
}
if reached.output {
result.OutputPath = paths.output
}
return result
}
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
t.Helper()
artifact, err := store.LoadPromptExecution(context.Background(), path)
if err != nil {
t.Fatalf("LoadPromptExecution() error = %v", err)
}
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
}
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
}
if artifact.Paths != want {
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
}
}
type promptArtifactPaths struct {
state.ArtifactPaths
output string
}
type reachedPromptArtifacts struct {
preparation bool
execution bool
metadata bool
raw bool
normalized bool
renderContext bool
report bool
output bool
}
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
}, mustParse("2026-05-29T05:00:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read daily fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(bundleData, &bundle); err != nil {
t.Fatalf("decode daily fixture: %v", err)
}
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystemStore.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
debugWriter, err := promptdebug.NewPromptDebugWriter("")
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
return promptReportRequest{
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
Inspection: PromptInspectionResult{
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
},
DebugWriter: debugWriter, noNotify: true,
}, promptArtifactPaths{ArtifactPaths: paths}
}
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
t.Helper()
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
}
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
}
checks := []struct {
name string
got string
metadataGot string
inMetadata bool
path string
want bool
}{
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
{"output", result.OutputPath, "", false, paths.output, want.output},
}
for _, check := range checks {
if check.want && check.got != check.path {
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
}
if check.want && check.inMetadata && check.metadataGot != check.path {
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
}
if !check.want && check.got != "" {
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
}
if !check.want && check.inMetadata && check.metadataGot != "" {
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
}
}
}

View File

@@ -3,7 +3,6 @@ package app
import ( import (
"context" "context"
"fmt" "fmt"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
@@ -14,7 +13,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
) )
type promptReportRequest struct { type promptReportRequest struct {
@@ -23,9 +22,23 @@ type promptReportRequest struct {
Collection collect.Result Collection collect.Result
Inspection PromptInspectionResult Inspection PromptInspectionResult
DebugWriter *promptdebug.PromptDebugWriter DebugWriter *promptdebug.PromptDebugWriter
Result *ReportResult
noNotify bool noNotify bool
} }
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
result *ReportResult
briefingMetadata briefing.Metadata
reportFacts ReportFacts
moduleSnapshot module.Snapshot
dataPackage []byte
handler generatedtext.Handler
debugRef promptdebug.PromptDebugRef
callbackFailed bool
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
workflow, err := newPromptReportWorkflow(ctx, req) workflow, err := newPromptReportWorkflow(ctx, req)
if err != nil { if err != nil {
@@ -34,369 +47,145 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
if err := workflow.buildInputs(); err != nil { if err := workflow.buildInputs(); err != nil {
return workflow.result, err return workflow.result, err
} }
execution, err := workflow.executePrompt()
execution, executeErr := workflow.executePrompt() if err != nil {
if executeErr != nil { if workflow.callbackFailed {
return workflow.result, workflow.handleExecutionFailure(executeErr) return workflow.result, err
}
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
} }
if execution == nil { if execution == nil {
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil) return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
} }
return workflow.result, workflow.reportError("execute prompt", err) workflow.result.ValidationStatus = execution.Validation.Status
} if err := workflow.writeExecutionDebug(*execution); err != nil {
if err := workflow.persistExecutionDebug(*execution); err != nil {
return workflow.result, err return workflow.result, err
} }
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil) return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
}
return workflow.result, workflow.reportError("validate prompt execution", err)
}
if err := workflow.persistCompletedExecution(*execution); err != nil {
return workflow.result, err
} }
if execution.Validation.Status == promptexec.ValidationFailed { if execution.Validation.Status == promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil) return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
return workflow.result, workflow.reportError("validate prompt execution", err)
} }
return workflow.renderAndPublish(execution.RawOutput)
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
if err != nil {
return workflow.result, err
}
return workflow.finalizeReport(rendered)
}
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
store state.Store
result *ReportResult
metadata state.Metadata
briefingMetadata briefing.Metadata
reportFacts ReportFacts
moduleSnapshot module.Snapshot
dataPackageBytes []byte
handler generatedtext.Handler
executionArtifact state.PromptExecutionArtifact
debugRef promptdebug.PromptDebugRef
prepared bool
callbackFailed bool
} }
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) { func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
if req.Collection.Bundle == nil { if req.Collection.Bundle == nil {
return nil, fmt.Errorf("collected weather bundle is required") return nil, fmt.Errorf("collected weather bundle is required")
} }
store := req.Store result := req.Result
var err error if result == nil {
if store == nil { result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
store, err = defaultStore(req.Config)
if err != nil {
return nil, err
} }
return &promptReportWorkflow{
ctx: ctx, req: req,
result: result,
}, nil
}
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
metadata := resolved.Metadata()
return &ReportResult{
ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name,
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone,
ValidPeriod: metadata.ValidPeriod,
ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName,
} }
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
} }
func (w *promptReportWorkflow) buildInputs() error { func (w *promptReportWorkflow) buildInputs() error {
paths, err := w.store.Paths(w.req.Resolved) var err error
if err != nil {
return err
}
w.result = &ReportResult{}
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle) w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
if err != nil { if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err) return w.reportError("build report facts", err)
} }
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts) w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
if err != nil { if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err) return w.reportError("build module snapshot", err)
} }
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
if err != nil {
return err
}
w.result.ModuleSnapshot = w.moduleSnapshot
w.result.ModuleSnapshotPath = moduleSnapshotPath
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected)) w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{ w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
ModuleSnapshot: moduleSnapshotPath, dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
Metadata: paths.Metadata,
})
w.result.Metadata = w.metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot,
})
if err != nil { if err != nil {
return w.reportError("build data package", err) return w.reportError("build data package", err)
} }
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage) w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
if err != nil { if err != nil {
return err return w.reportError("marshal data package", err)
} }
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
if err != nil {
return err
}
w.metadata.DataPackagePath = dataPackagePath
w.result.DataPackage = dataPackage
w.result.DataPackagePath = dataPackagePath
w.result.Metadata = w.metadata
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition) w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
if err != nil { if err != nil {
return w.reportError("lookup generated text catalog", err) return w.reportError("lookup generated text catalog", err)
} }
w.debugRef = promptdebug.PromptDebugRef{ w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
}
return nil return nil
} }
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) { func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{ captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
}, w.persistPreparation)
} }
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{ w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded, if w.req.DebugWriter == nil {
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID, return nil
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
} }
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact) path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
if err != nil {
w.callbackFailed = true
return err
}
w.prepared = true
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
if err != nil { if err != nil {
w.callbackFailed = true w.callbackFailed = true
return promptDebugWriteError(err) return promptDebugWriteError(err)
} }
if debugPath != "" { w.result.LLMDebugPath = path
w.result.LLMDebugPath = debugPath
}
if err := w.saveMetadata(); err != nil {
w.callbackFailed = true
return err
}
return nil return nil
} }
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error { func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
if w.callbackFailed { if w.req.DebugWriter == nil {
return executeErr return nil
} }
executeErr = classifiedPromptError("prompt execution failed", executeErr) path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
if !w.prepared {
if err := w.persistPreparationFailure(executeErr); err != nil {
return err
}
return w.reportError("prepare prompt", executeErr)
}
if promptexec.CategoryOf(executeErr) != "" {
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
return err
}
}
return w.reportError("execute prompt", executeErr)
}
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
startedAt, endedAt := time.Now(), time.Now()
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
Error: state.NewPromptArtifactError(executeErr),
}
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.ExecutionPath = path
w.metadata.ExecutionPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
if err != nil { if err != nil {
return w.reportError("write prompt debug", promptDebugWriteError(err)) return w.reportError("write prompt debug", promptDebugWriteError(err))
} }
if debugPath != "" { if path != "" {
w.result.LLMDebugPath = debugPath w.result.LLMDebugPath = path
} }
return nil return nil
} }
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error { func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput) generatedText, _, err := w.handler.Validate(raw)
if err != nil { if err != nil {
return err return w.result, w.reportError("validate generated text", err)
} }
w.result.GeneratedTextRawPath = rawPath
w.metadata.GeneratedTextRawPath = rawPath
w.result.Metadata = w.metadata
w.executionArtifact = state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
}
if execution.Validation.Status == promptexec.ValidationPassed {
w.executionArtifact.Status = state.PromptExecutionSucceeded
} else {
w.executionArtifact.Status = state.PromptExecutionValidationRejected
}
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
if err != nil {
return err
}
w.result.ExecutionPath = executionPath
w.metadata.ExecutionPath = executionPath
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
generatedText, normalized, err := w.handler.Validate(raw)
if err != nil {
return nil, w.reportError("validate generated text", err)
}
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
if err != nil {
return nil, err
}
w.result.GeneratedTextPath = generatedTextPath
w.metadata.GeneratedTextPath = generatedTextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
return nil, err
}
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText) renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
if err != nil { if err != nil {
return nil, w.reportError("build render context", err) return w.result, w.reportError("build render context", err)
} }
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
if err != nil {
return nil, err
}
w.result.RenderContextPath = renderContextPath
w.metadata.RenderContextPath = renderContextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
return nil, err
}
rendered, err := w.handler.Render(renderContext) rendered, err := w.handler.Render(renderContext)
if err != nil { if err != nil {
return nil, w.reportError("render template", err) return w.result, w.reportError("render template", err)
} }
return rendered, nil if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
} return w.result, err
}
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) { w.result.OutputPath = w.req.OutputPath
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved) if w.req.noNotify {
return w.result, nil
}
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
w.result.Notification = notification
if err != nil { if err != nil {
return w.result, err return w.result, err
} }
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil { return w.result, nil
return w.result, err
}
w.result.ReportPath = reportPath
w.metadata.RenderedReportPath = reportPath
w.result.Metadata = w.metadata
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
return w.result, err
}
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
ExecutionArtifact: &w.executionArtifact, RenderedReportPath: reportPath, OutputPath: w.req.OutputPath,
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
})
w.result.OutputPath = finalized.OutputPath
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
return w.result, err
}
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
}
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
if err := w.persistReachedPath(update); err != nil {
return err
}
return w.saveMetadata()
}
func (w *promptReportWorkflow) saveMetadata() error {
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
if err != nil {
return err
}
w.result.Metadata = w.metadata
w.result.MetadataPath = path
return nil
} }
func (w *promptReportWorkflow) reportError(operation string, err error) error { func (w *promptReportWorkflow) reportError(operation string, err error) error {
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err) return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
}
func persistReachedPromptPath(
ctx context.Context,
store state.Store,
resolved report.Resolved,
artifact *state.PromptExecutionArtifact,
update func(*state.PromptExecutionPaths),
) error {
update(&artifact.Paths)
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
return err
}
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
now := time.Now()
return state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed,
ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now,
Error: state.NewPromptArtifactError(err),
}
} }
func classifiedPromptError(operation string, err error) error { func classifiedPromptError(operation string, err error) error {
@@ -409,5 +198,3 @@ func classifiedPromptError(operation string, err error) error {
func promptDebugWriteError(err error) error { func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err) return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
} }
func ptr[T any](value T) *T { return &value }

View File

@@ -1,671 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type workflowCollector struct {
result *collect.Result
err error
calls int
}
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
c.calls++
if c.err != nil {
return nil, c.err
}
return c.result, nil
}
type workflowExecutor struct {
definition report.Definition
raw []byte
prompt promptexec.PromptInspection
inspectionErr error
profile promptexec.ProfileInspection
profileErr error
beforePreparationErr error
afterCallbackErr error
afterPreparationErr error
validation promptexec.ValidationStatus
executeCalls int
providerCalls int
request promptexec.ExecuteRequest
beforeProvider func()
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
preparation *promptexec.Preparation
execution *promptexec.Execution
}
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
if e.inspectionErr != nil {
return promptexec.PromptInspection{}, e.inspectionErr
}
if id != e.definition.PromptID || version != e.definition.PromptVersion {
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
}
if e.prompt.PromptID != "" {
return e.prompt, nil
}
return validPromptInspection(e.definition), nil
}
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
if e.profileErr != nil {
return promptexec.ProfileInspection{}, e.profileErr
}
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
}
return profile, nil
}
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
e.executeCalls++
e.request = req
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"}
}
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
e.preparation = &preparation
if err := callback(preparation, e.preparationDebug); err != nil {
return nil, err
}
if e.afterCallbackErr != nil {
return nil, e.afterCallbackErr
}
if e.beforeProvider != nil {
e.beforeProvider()
}
e.providerCalls++
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
execution := &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
Debug: e.executionDebug,
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}
e.execution = execution
return execution, nil
}
type workflowNotifier struct {
requests []NotificationRequest
err error
}
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.requests = append(n.requests, req)
if n.err != nil {
return nil, n.err
}
return &NotificationResult{
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
date time.Time
raw string
wantOutput string
}{
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(test.id)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
outputPath := filepath.Join(t.TempDir(), test.name+".md")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
}
if executor.request.PromptVersion != definition.PromptVersion {
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
}
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
if storeErr != nil {
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
}
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
}
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
}
persisted, readErr := os.ReadFile(result.DataPackagePath)
if readErr != nil {
t.Fatalf("read data package: %v", readErr)
}
if !bytes.Equal(executor.request.DataPackage, persisted) {
t.Fatal("executor data package differs from exact persisted YAML bytes")
}
renderedReport, readErr := os.ReadFile(result.ReportPath)
if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) {
t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput)
}
copied, readErr := os.ReadFile(outputPath)
if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath {
t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath)
}
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath {
t.Fatalf("notification requests = %#v, want selected output source", notifier.requests)
}
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
if notifier.requests[0].PipelineID != wantPipeline {
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
}
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
}
renderedName := filepath.Base(result.ReportPath)
if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath)
}
})
}
}
func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
raw string
override string
profile promptexec.ProfileInspection
}{
{
name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"},
},
{
name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
},
{
name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile",
profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Promptkit.Profile = test.override
definition := report.DefaultRegistry().MustLookup(test.id)
executor := &workflowExecutor{
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
}
bundle := workflowBundle(t)
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if executor.request.ProfileID != test.profile.ProfileID {
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
}
if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName {
t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName {
t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
})
}
}
type preparationFailingStore struct {
state.Store
}
type renderContextFailingStore struct {
state.Store
}
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
if err != nil {
return "", err
}
for index := range snapshot.Outputs {
snapshot.Outputs[index].Value = "invalid module value"
}
return path, nil
}
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
return "", errors.New("injected preparation persistence failure")
}
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
wantPreparation bool
wantExecution bool
wantRaw bool
wantProviderCall int
}{
{name: "preparation", configure: func(e *workflowExecutor) {
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true},
{name: "credential disappears", configure: func(e *workflowExecutor) {
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "canceled", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "deadline", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "generation", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation error", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
test.configure(executor)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: collector, Executor: executor,
})
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
}
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
}
})
}
}
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
}{
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
{name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration},
{name: "malformed profile", configure: func(e *workflowExecutor) {
e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret")
}, wantCategory: promptexec.InvalidConfiguration},
{name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration},
{name: "credential", configure: func(e *workflowExecutor) {
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
}, wantCategory: promptexec.MissingCredential},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition}
test.configure(executor)
collector := &workflowCollector{err: errors.New("collector must not run")}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: collector, Executor: executor,
})
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
}
if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") {
t.Fatalf("error leaks profile details: %v", err)
}
entries, readErr := os.ReadDir(cfg.Workspace.Root)
if readErr != nil || len(entries) != 0 {
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
}
})
}
}
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
})
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
}
}
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystem.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
checked := false
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
executor.beforeProvider = func() {
checked = true
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
}
}
bundle := workflowBundle(t)
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.Store = filesystem
result, err := GenerateDetailed(context.Background(), request)
if err != nil || result == nil || !checked || result.OutputPath == "" {
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
}
}
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
tests := []struct {
name string
raw string
configure func(*GenerateRequest, *workflowNotifier)
wantRaw bool
wantNormalized bool
wantContext bool
wantReport bool
wantOutput bool
}{
{name: "generated text decode", raw: `{`, wantRaw: true},
{name: "generated text domain", raw: `{}`, wantRaw: true},
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = renderContextFailingStore{Store: req.Store}
}, wantRaw: true, wantNormalized: true},
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
}, wantRaw: true, wantNormalized: true},
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
blocker := filepath.Join(t.TempDir(), "report-blocker")
if err := os.Mkdir(blocker, 0o700); err != nil {
t.Fatalf("create report blocker: %v", err)
}
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
}, wantRaw: true, wantNormalized: true, wantContext: true},
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
notifier.err = errors.New("notification rejected")
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
bundle := workflowBundle(t)
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
req := GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: executor, Notifier: notifier, Store: filesystem,
}
if test.configure != nil {
test.configure(&req, notifier)
}
result, err := GenerateDetailed(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
}
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
(result.OutputPath != "") != test.wantOutput {
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath)
}
if test.wantRaw {
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
}
}
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) {
t.Fatalf("notification requests = %#v", notifier.requests)
}
if test.name == "notification" && result.Metadata.NotificationPath != "" {
t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata)
}
})
}
}
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
tests := []struct {
name string
createCollision func(string, report.Resolved) error
wantProviderCalls int
wantPreparationFile bool
}{
{
name: "preparation debug",
createCollision: func(root string, resolved report.Resolved) error {
path := workflowDebugRunPath(root, resolved)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
return os.WriteFile(path, []byte("not a directory"), 0o600)
},
},
{
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
createCollision: func(root string, resolved report.Resolved) error {
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
if err := test.createCollision(debugRoot, resolved); err != nil {
t.Fatalf("create debug collision: %v", err)
}
bundle := workflowBundle(t)
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.LLMDebugDir = debugRoot
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
}
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if test.wantPreparationFile && result.MetadataPath == "" {
t.Fatal("execution debug failure lost previously persisted metadata")
}
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
_, statErr := os.Stat(preparationDebug)
if (statErr == nil) != test.wantPreparationFile {
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
}
})
}
}
func workflowDebugRunPath(root string, resolved report.Resolved) string {
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
}
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
switch id {
case report.Daily:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
case report.Today:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
case report.Tomorrow:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
case report.Hourly:
return []string{"hourly/index.md"}
default:
return nil
}
}
func workflowConfig(t *testing.T) config.Config {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
cfg.WeatherAPI.Timezone = "America/Chicago"
cfg.Location.ID = "home"
cfg.Location.Name = "Testville"
cfg.Location.Region = "MO"
cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
return cfg
}
func workflowBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
future := bundle.Hourly.Periods[0]
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
futureNarrative := bundle.Narrative.Periods[0]
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
futureNarrative.Name = "Tomorrow"
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
return bundle
}
func workflowTime(value string) time.Time {
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
panic(err)
}
return parsed
}
func validHourlyWorkflowJSON() string {
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`
}
func validTomorrowWorkflowJSON() string {
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validTodayWorkflowJSON() string {
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validDailyWorkflowJSON() string {
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
}

View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/app" "gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
) )
const ( const (
@@ -25,16 +26,15 @@ type generateSummary struct {
Status string `json:"status"` Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"` GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"` ValidPeriod timeutil.Period `json:"validPeriod"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"` OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"` LLMDebugPath string `json:"llmDebugPath,omitempty"`
GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"` PromptVersion string `json:"promptVersion"`
GeneratedTextPath string `json:"generatedTextPath,omitempty"` Timezone string `json:"timezone"`
RenderContextPath string `json:"renderContextPath,omitempty"` ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus string `json:"validationStatus,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"` Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
} }
@@ -73,24 +73,20 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
return summary return summary
} }
metadata := result.Metadata summary.ReportID = result.ReportID
summary.ReportID = metadata.ReportID summary.ReportName = result.ReportName
summary.ReportName = reportName(metadata.ReportID) summary.PromptID = result.PromptID
summary.PromptID = metadata.PromptID summary.PromptVersion = result.PromptVersion
summary.RunID = metadata.RunID summary.RunID = result.RunID
summary.Status = summaryStatusSucceeded summary.Status = summaryStatusSucceeded
summary.GeneratedAt = metadata.GeneratedAt summary.GeneratedAt = result.GeneratedAt
summary.ValidPeriod = metadata.ValidPeriod summary.ValidPeriod = result.ValidPeriod
summary.ReportPath = result.ReportPath summary.Timezone = result.Timezone
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
summary.ValidationStatus = string(result.ValidationStatus)
summary.OutputPath = result.OutputPath summary.OutputPath = result.OutputPath
summary.MetadataPath = result.MetadataPath
summary.DataPackagePath = result.DataPackagePath
summary.PreparationPath = result.PreparationPath
summary.ExecutionPath = result.ExecutionPath
summary.LLMDebugPath = result.LLMDebugPath summary.LLMDebugPath = result.LLMDebugPath
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
summary.GeneratedTextPath = result.GeneratedTextPath
summary.RenderContextPath = result.RenderContextPath
summary.Notification = newGenerateNotificationSummary(result.Notification) summary.Notification = newGenerateNotificationSummary(result.Notification)
if err != nil { if err != nil {
summary.Status = summaryStatusFailed summary.Status = summaryStatusFailed

View File

@@ -2,264 +2,29 @@ package cli
import ( import (
"encoding/json" "encoding/json"
"errors"
"strings" "strings"
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app" "gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
) )
func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) { func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC) generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
acceptedAt := generatedAt.Add(time.Minute) summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
startedAt := acceptedAt.Add(time.Minute) if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) {
finishedAt := startedAt.Add(time.Minute) t.Fatalf("summary = %#v", summary)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json",
GeneratedTextPath: "/runs/hourly/generated_text.json",
RenderContextPath: "/runs/hourly/render_context.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
Notification: &app.NotificationResult{
Status: "succeeded",
UploadStatus: "accepted",
RunID: "distributor-run",
PipelineID: "weatherreporter.hourly",
BundleID: "weatherreporter.home.hourly",
IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly",
AcceptedAt: acceptedAt,
StartedAt: &startedAt,
FinishedAt: &finishedAt,
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
},
}
summary := newGenerateSummary(result, nil)
if summary.Command != "generate" || summary.Status != "succeeded" {
t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status)
}
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {
t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification)
} }
data, err := json.Marshal(summary) data, err := json.Marshal(summary)
if err != nil { if err != nil {
t.Fatalf("Marshal() error = %v", err) t.Fatalf("Marshal() error = %v", err)
} }
if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") { for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data)) if strings.Contains(string(data), forbidden) {
} t.Fatalf("summary includes %q: %s", forbidden, data)
if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") {
t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data))
}
}
func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
ReportPath: "/runs/daily/report.md",
OutputPath: "/copies/daily.md",
MetadataPath: "/runs/daily/metadata.json",
Metadata: state.Metadata{
ReportID: report.Daily,
PromptID: "weather.daily_generated_text",
RunID: "20260529T133000Z_daily",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
summary := newGenerateSummary(result, nil)
if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" {
t.Fatalf("summary = %#v, want successful daily summary", summary)
}
if summary.Notification != nil {
t.Fatalf("notification summary = %#v, want omitted", summary.Notification)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
for _, omitted := range []string{"notification"} {
if strings.Contains(string(data), omitted) {
t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data))
} }
} }
} }
func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) {
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
Metadata: state.Metadata{
ReportID: report.Daily,
RunID: "20260529T133000Z_daily",
},
}
data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed")))
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("partial summary omits reached paths:\n%s", text)
}
}
func TestNewGenerateSummaryForNotificationFailure(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`)
summary := newGenerateSummary(result, err)
if summary.Status != "failed" || summary.Error != err.Error() {
t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error)
}
if summary.ReportPath == "" || summary.MetadataPath == "" {
t.Fatalf("artifact paths = report %q metadata %q, want retained output provenance", summary.ReportPath, summary.MetadataPath)
}
data, marshalErr := json.Marshal(summary)
if marshalErr != nil {
t.Fatalf("Marshal() error = %v", marshalErr)
}
if strings.Contains(string(data), "notificationPath") {
t.Fatalf("summary JSON includes a notification receipt path:\n%s", data)
}
}
func TestNewBatchSummaryStatusDerivation(t *testing.T) {
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
finishedAt := startedAt.Add(2 * time.Minute)
tests := []struct {
name string
result *app.BatchResult
wantStatus string
wantError string
}{
{
name: "success",
result: &app.BatchResult{
Batch: app.BatchMorning,
StartedAt: startedAt,
FinishedAt: finishedAt,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}},
},
wantStatus: "succeeded",
},
{
name: "report failure",
result: &app.BatchResult{
Batch: app.BatchMorning,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{
{ReportID: report.Today, Status: "succeeded"},
{ReportID: report.Tomorrow, Status: "failed", Error: "render failed"},
},
},
wantStatus: "failed",
wantError: "batch morning failed: 1 of 2 reports failed",
},
{
name: "skipped notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}},
Notification: &app.BatchNotificationResult{
Status: "skipped",
Reason: "one or more reports failed",
},
},
wantStatus: "failed",
wantError: "batch evening failed: 1 of 2 reports failed",
},
{
name: "failed notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}},
Notification: &app.BatchNotificationResult{
Status: "failed",
Error: "notify batch evening: upload rejected",
},
},
wantStatus: "failed",
wantError: "batch evening notification failed: notify batch evening: upload rejected",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := newBatchSummary(tt.result)
if summary.Command != "run" || summary.Status != tt.wantStatus {
t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus)
}
if summary.Error != tt.wantError {
t.Fatalf("error = %q, want %q", summary.Error, tt.wantError)
}
if len(summary.Reports) != len(tt.result.Reports) {
t.Fatalf("reports = %#v, want copied report list", summary.Reports)
}
})
}
}
func testSummaryPeriod(start time.Time) timeutil.Period {
return timeutil.Period{
Start: start,
End: start.Add(6 * time.Hour),
}
}

View File

@@ -1,631 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
const (
testRenderedPrompt = "PRIVATE RENDERED PROMPT"
testSchemaBody = `{"private":"schema"}`
testDataBody = "PRIVATE DATA PACKAGE"
testGeneratedBody = "PRIVATE GENERATED BODY"
testEndpoint = "https://user:credential@example.invalid/v1?token=credential"
testParameters = `{"temperature":0.2,"private":"parameter"}`
testCredential = "cli-secret-credential"
)
type commandOutput struct {
stdout string
stderr string
}
type cliExecutor struct {
fail bool
failPrompt string
}
func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text")
return promptexec.PromptInspection{
PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile",
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"},
}, nil
}
func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil
}
func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath,
StartedAt: now, EndedAt: now,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if e.fail || req.PromptID == e.failPrompt {
return nil, errors.New(strings.Join([]string{
"provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters,
}, " "))
}
raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`)
if req.PromptID == "weather.hourly_generated_text" {
raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`)
}
return &promptexec.Execution{
RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil),
}, nil
}
func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) {
output, err := runCLICommand(Runner{}, "--help")
if err != nil {
t.Fatalf("Run(--help) error = %v", err)
}
for _, command := range []string{
"--version",
"generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening",
"inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources",
} {
if !strings.Contains(output.stdout, command) {
t.Fatalf("help missing %q:\n%s", command, output.stdout)
}
}
for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} {
if strings.Contains(output.stdout, retired) {
t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout)
}
}
for _, description := range []string{"Write the generated Markdown report to PATH.", "Write generated Markdown reports beneath PATH"} {
if !strings.Contains(output.stdout, description) {
t.Fatalf("help missing output description %q:\n%s", description, output.stdout)
}
}
}
func TestRunnerVersion(t *testing.T) {
for _, test := range []struct {
name string
runner Runner
version string
}{
{name: "development default", runner: Runner{}, version: "development"},
{name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"},
} {
t.Run(test.name, func(t *testing.T) {
output, err := runCLICommand(test.runner, "--version")
if err != nil {
t.Fatalf("Run(--version) error = %v", err)
}
if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" {
t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr)
}
})
}
if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil {
t.Fatal("Run(--version extra) error = nil")
}
}
func TestResolveSupportedCommandsAndFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, constructions := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want app.ReportKind
}{
{name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily},
{name: "today", args: []string{"today"}, want: app.ReportToday},
{name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow},
{name: "hourly", args: []string{"hourly"}, want: app.ReportHourly},
} {
t.Run(tt.name, func(t *testing.T) {
args := append(tt.args, "--config", configPath)
req, err := runner.resolveGenerate(args)
if err != nil {
t.Fatalf("resolveGenerate() error = %v", err)
}
if req.Report != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want report %q with executor", req, tt.want)
}
if tt.want == app.ReportDaily || tt.want == app.ReportToday {
if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" {
t.Fatalf("resolved date = %q, want 2026-05-29", got)
}
} else if !req.Date.IsZero() {
t.Fatalf("resolved date = %s, want unset", req.Date)
}
})
}
for _, tt := range []struct {
name string
want app.BatchKind
}{
{name: "morning", want: app.BatchMorning},
{name: "evening", want: app.BatchEvening},
} {
t.Run(tt.name, func(t *testing.T) {
req, err := runner.resolveRun([]string{tt.name, "--config", configPath})
if err != nil {
t.Fatalf("resolveRun() error = %v", err)
}
if req.Batch != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want batch %q with executor", req, tt.want)
}
})
}
if *constructions != 6 {
t.Fatalf("executor constructions = %d, want one per resolved action", *constructions)
}
}
func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
generate, generateOpts, err := runner.resolveGenerateAction([]string{
"daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC",
"--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveGenerateAction() error = %v", err)
}
if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != filepath.Join(runner.WorkingDir, "daily.md") || generate.WorkingDir != runner.WorkingDir || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet {
t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts)
}
if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" {
t.Fatalf("generate date = %q, want 2026-05-30", got)
}
batch, batchOpts, err := runner.resolveRunAction([]string{
"evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports",
"--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveRunAction() error = %v", err)
}
if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != filepath.Join(runner.WorkingDir, "reports") || batch.WorkingDir != runner.WorkingDir || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet {
t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts)
}
}
func TestGenerateFlagContracts(t *testing.T) {
for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(kind), func(t *testing.T) {
opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"})
if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" {
t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err)
}
})
}
runner, _ := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want string
}{
{name: "daily requires date", args: []string{"daily"}, want: "requires --date"},
{name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}},
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}},
{name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}},
{name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}},
{name: "batch rejects output", args: []string{"run", "--out", "report.md"}},
} {
t.Run(tt.name, func(t *testing.T) {
var err error
if tt.args[0] == "run" {
_, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...))
} else {
_, err = runner.resolveGenerate(tt.args)
}
if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) {
t.Fatalf("error = %v, want rejection containing %q", err, tt.want)
}
})
}
}
func TestResolversRejectRetiredAndUnknownNames(t *testing.T) {
runner, _ := countingRunner(cliExecutor{})
for _, name := range []string{"near-term", "three-day", "weekend", "storm"} {
if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") {
t.Fatalf("resolveGenerate(%q) error = %v", name, err)
}
}
for _, name := range []string{"daily", "weekend", "storm"} {
if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") {
t.Fatalf("resolveRun(%q) error = %v", name, err)
}
}
}
func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) {
for _, tt := range []struct {
name string
args func(string, string) []string
}{
{name: "single", args: func(configPath, outputPath string) []string {
return []string{"generate", "today", "--config", configPath, "--out", outputPath}
}},
{name: "batch", args: func(configPath, outputPath string) []string {
return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath}
}},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
assertRoutineOutputSafe(t, output)
if tt.name == "single" {
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" {
t.Fatalf("single summary = %#v", summary)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout)
}
} else {
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" {
t.Fatalf("batch summary = %#v", summary)
}
if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") {
t.Fatalf("batch status = %q", output.stderr)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout)
}
}
})
}
}
func TestRunnerPreRunFailureAndQuietMode(t *testing.T) {
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "daily")
if err == nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions)
}
fixture := newCLIFixture(t)
runner, _ = countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet")
if err != nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("quiet output/error = %#v/%v", output, err)
}
}
func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{fail: true})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want execution failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" {
t.Fatalf("failed summary paths = %#v", summary)
}
if !strings.Contains(summary.Error, "prompt execution failed") {
t.Fatalf("failed summary error = %q", summary.Error)
}
assertRoutineOutputSafe(t, output)
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID)
if inspectErr != nil {
t.Fatalf("inspect %s error = %v", command, inspectErr)
}
if !strings.Contains(inspected.stdout, summary.RunID) {
t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout)
}
assertRoutineOutputSafe(t, inspected)
}
}
func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want aggregate batch failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 {
t.Fatalf("failed batch summary = %#v", summary)
}
var succeeded, failed *app.BatchReportResult
for index := range summary.Reports {
item := &summary.Reports[index]
if item.Status == summaryStatusSucceeded {
succeeded = item
} else if item.Status == summaryStatusFailed {
failed = item
}
}
if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" {
t.Fatalf("successful batch item paths = %#v", succeeded)
}
if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" {
t.Fatalf("failed batch item paths = %#v", failed)
}
if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") {
t.Fatalf("partial batch status = %q", output.stderr)
}
assertRoutineOutputSafe(t, output)
}
func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC))
second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC))
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2")
if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) {
t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err)
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID)
if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) {
t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr)
}
assertRoutineOutputSafe(t, output)
}
prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID)
if err != nil || !strings.Contains(prior.stdout, first.RunID) {
t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err)
}
}
func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runIDs := []string{
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)),
}
runID := runIDs[0]
runner, _ := countingRunner(cliExecutor{})
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath)
if err != nil {
t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err)
}
for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} {
if !strings.Contains(listed.stdout, want) {
t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout)
}
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID)
if inspectErr != nil || !strings.Contains(output.stdout, runID) {
t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr)
}
}
metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID)
if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) {
t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err)
}
}
func countingRunner(executor promptexec.Executor) (Runner, *int) {
count := new(int)
return Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
*count++
return executor, nil
},
}, count
}
func runCLICommand(runner Runner, args ...string) (commandOutput, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
err := runner.Run(context.Background(), args, &stdout, &stderr)
return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err
}
func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary {
t.Helper()
base.Clock = timeutil.FixedClock{Time: now}
output, err := runCLICommand(base, "generate", "today", "--config", configPath)
if err != nil {
t.Fatalf("generate current report: %v", err)
}
return decodeGenerateSummary(t, output.stdout)
}
func decodeGenerateSummary(t *testing.T, text string) generateSummary {
t.Helper()
var summary generateSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode generate summary: %v\n%s", err, text)
}
return summary
}
func decodeBatchSummary(t *testing.T, text string) batchSummary {
t.Helper()
var summary batchSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode batch summary: %v\n%s", err, text)
}
return summary
}
func assertRoutineOutputSafe(t *testing.T, output commandOutput) {
t.Helper()
combined := output.stdout + output.stderr
for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} {
if strings.Contains(combined, forbidden) {
t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined)
}
}
}
type cliFixture struct {
tempDir string
workspaceRoot string
configPath string
}
func newCLIFixture(t *testing.T) cliFixture {
t.Helper()
tempDir := t.TempDir()
workspaceRoot := filepath.Join(tempDir, "workspace")
server := weatherServer(t)
return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")}
}
func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) }
func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string {
t.Helper()
configPath := filepath.Join(t.TempDir(), "config.yml")
body := "weather_api:\n timezone: America/Chicago\n"
if baseURL != "" {
body += " base_url: " + baseURL + "\n"
}
body += "workspace:\n root: " + workspaceRoot + "\n"
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
return configPath
}
func weatherServer(t *testing.T) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/observations":
_, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`))
case "/conditions/current":
_, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`))
case "/forecast/hourly":
_, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`))
case "/forecast/narrative":
_, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`))
case "/alerts/active":
_, _ = w.Write([]byte(`{"data":{"alerts":[]}}`))
case "/discussion":
_, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`))
case "/weatherstories/latest":
_, _ = w.Write([]byte(`{"data":null}`))
case "/outlooks/convective":
_, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(server.Close)
return server
}
func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string {
t.Helper()
runID := "historical-" + string(reportID)
date := generatedAt.Format(timeutil.DateLayout)
dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date)
modulePath := filepath.Join(dir, "modules."+runID+".json")
dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml")
metadataPath := filepath.Join(dir, "metadata."+runID+".json")
if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil {
t.Fatalf("create historical fixture directory: %v", err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create historical metadata directory: %v", err)
}
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}})
if err != nil {
t.Fatalf("build historical module snapshot: %v", err)
}
moduleData, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal historical module snapshot: %v", err)
}
if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil {
t.Fatalf("write historical module snapshot: %v", err)
}
period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}
pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{
RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period,
}, Modules: snapshot})
if err != nil {
t.Fatalf("build historical data package: %v", err)
}
data, err := promptinput.MarshalYAML(pkg)
if err != nil {
t.Fatalf("marshal historical data package: %v", err)
}
if err := os.WriteFile(dataPath, data, 0o600); err != nil {
t.Fatalf("write historical data package: %v", err)
}
metadata := state.Metadata{
SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID),
GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive",
ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json",
}
metadataData, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("marshal historical metadata: %v", err)
}
if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil {
t.Fatalf("write historical metadata: %v", err)
}
return runID
}

View File

@@ -16,8 +16,8 @@ import (
) )
const ( const (
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v1" promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v1" promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
debugDirectoryMode = 0o700 debugDirectoryMode = 0o700
debugFileMode = 0o600 debugFileMode = 0o600
) )
@@ -64,7 +64,6 @@ type PromptDebugPreparation struct {
StartedAt time.Time `json:"startedAt"` StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"` EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"` Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
} }
// PromptPreparationDebugArtifact is the on-disk preparation debug record. // PromptPreparationDebugArtifact is the on-disk preparation debug record.
@@ -114,7 +113,6 @@ type PromptDebugExecution struct {
StartedAt time.Time `json:"startedAt"` StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"` EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"` Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
} }
// PromptExecutionDebugArtifact is the on-disk execution debug record. // PromptExecutionDebugArtifact is the on-disk execution debug record.
@@ -340,7 +338,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes), RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName, ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath}, Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath, StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
} }
} }
@@ -351,7 +349,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID, InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens}, Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath, StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
} }
} }

View File

@@ -40,19 +40,19 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir) t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
} }
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json")) preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} { for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
if !strings.Contains(string(preparationData), want) { if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData) t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
} }
} }
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json")) executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} { for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
if !strings.Contains(string(executionData), want) { if !strings.Contains(string(executionData), want) {
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData) t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
} }
} }
for _, data := range [][]byte{preparationData, executionData} { for _, data := range [][]byte{preparationData, executionData} {
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") { if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") {
t.Fatalf("debug artifact contains credentials:\n%s", data) t.Fatalf("debug artifact contains credentials:\n%s", data)
} }
} }
@@ -169,7 +169,7 @@ func promptDebugPreparationFixture() promptexec.Preparation {
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"}, Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml", StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
} }
} }
@@ -181,7 +181,7 @@ func promptDebugExecutionFixture() promptexec.Execution {
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}, GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."), RawOutput: []byte("Generated forecast prose."),
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}}, Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
} }
} }

View File

@@ -61,14 +61,12 @@ type ProfileInspection struct {
} }
// ExecuteRequest selects one exact prompt execution. DataPackage is the exact // ExecuteRequest selects one exact prompt execution. DataPackage is the exact
// YAML input; implementations must copy it before retaining it. DataPackagePath // YAML input; implementations must copy it before retaining it.
// is provenance for the inline input, not a provider-readable file reference.
type ExecuteRequest struct { type ExecuteRequest struct {
PromptID string PromptID string
PromptVersion string PromptVersion string
ProfileID string ProfileID string
DataPackage []byte DataPackage []byte
DataPackagePath string
CaptureDebug bool CaptureDebug bool
} }
@@ -90,7 +88,6 @@ type Preparation struct {
StartedAt time.Time StartedAt time.Time
EndedAt time.Time EndedAt time.Time
Duration time.Duration Duration time.Duration
DataPackagePath string
} }
// PreparationDebug contains content-rich preparation details for an explicitly // PreparationDebug contains content-rich preparation details for an explicitly
@@ -127,7 +124,6 @@ type Execution struct {
EndedAt time.Time EndedAt time.Time
Duration time.Duration Duration time.Duration
Validation Validation Validation Validation
DataPackagePath string
RawOutput []byte RawOutput []byte
Debug *ExecutionDebug Debug *ExecutionDebug
} }

View File

@@ -42,7 +42,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
if executor.operationalFailure != nil { if executor.operationalFailure != nil {
return nil, executor.operationalFailure return nil, executor.operationalFailure
} }
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath} preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion}
var debug *PreparationDebug var debug *PreparationDebug
if request.CaptureDebug { if request.CaptureDebug {
debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}} debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}}
@@ -57,7 +57,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
if executor.validationRejected { if executor.validationRejected {
status = ValidationFailed status = ValidationFailed
} }
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath, Validation: Validation{Status: status}, RawOutput: []byte("generated content")} result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
if request.CaptureDebug { if request.CaptureDebug {
result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")} result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")}
} }
@@ -65,7 +65,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
} }
func TestExecutorLifecycleFixtures(t *testing.T) { func TestExecutorLifecycleFixtures(t *testing.T) {
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", DataPackagePath: "data_package.yaml"} request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"}
t.Run("callback failure prevents provider execution", func(t *testing.T) { t.Run("callback failure prevents provider execution", func(t *testing.T) {
executor := &lifecycleExecutor{} executor := &lifecycleExecutor{}
callbackError := errors.New("persistence failed") callbackError := errors.New("persistence failed")
@@ -239,7 +239,6 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
BackendID: "local", BackendID: "local",
ModelName: "model-name", ModelName: "model-name",
Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"}, Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"},
DataPackagePath: "data-packages/daily/data_package.yaml",
} }
execution := Execution{ execution := Execution{
RunID: "run-id", RunID: "run-id",
@@ -254,7 +253,7 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
GeneratedHash: "generated-hash", GeneratedHash: "generated-hash",
RawOutput: []byte("generated content"), RawOutput: []byte("generated content"),
} }
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + preparation.DataPackagePath + execution.RunID + execution.GeneratedHash text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash
for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} { for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} {
if strings.Contains(text, unwanted) { if strings.Contains(text, unwanted) {
t.Fatalf("safe values contain %q: %s", unwanted, text) t.Fatalf("safe values contain %q: %s", unwanted, text)

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module" "gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
@@ -366,21 +367,40 @@ func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) Promp
artifact.DataPackagePath = paths.DataPackage artifact.DataPackagePath = paths.DataPackage
artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion
artifact.Preparation.PromptHash = "prompt-hash" artifact.Preparation.PromptHash = "prompt-hash"
artifact.Preparation.DataPackagePath = paths.DataPackage
return artifact return artifact
} }
func validPreparationArtifact() PromptPreparationArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptPreparationArtifact{
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
DataPackagePath: "/workspace/data.yaml", Preparation: &promptexec.Preparation{PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0"},
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact { func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact {
artifact := validExecutionArtifact() artifact := validExecutionArtifact()
metadata := resolved.Metadata() metadata := resolved.Metadata()
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion
artifact.Provenance.DataPackagePath = paths.DataPackage
artifact.Paths.RawOutputPath = paths.GeneratedTextRaw artifact.Paths.RawOutputPath = paths.GeneratedTextRaw
return artifact return artifact
} }
func validExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
Provenance: &PromptExecutionProvenance{RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", BackendID: "backend", ModelName: "model", StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second},
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func writeJSONFixture(t *testing.T, path string, value any) { func writeJSONFixture(t *testing.T, path string, value any) {
t.Helper() t.Helper()
data, err := json.Marshal(value) data, err := json.Marshal(value)

View File

@@ -81,7 +81,7 @@ func (a PromptPreparationArtifact) Validate() error {
if a.Preparation == nil || a.Error != nil { if a.Preparation == nil || a.Error != nil {
return fmt.Errorf("successful prompt preparation requires preparation without an error") return fmt.Errorf("successful prompt preparation requires preparation without an error")
} }
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath { if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion {
return fmt.Errorf("successful prompt preparation provenance must match the artifact") return fmt.Errorf("successful prompt preparation provenance must match the artifact")
} }
case PromptPreparationFailed: case PromptPreparationFailed:
@@ -122,7 +122,6 @@ type PromptExecutionProvenance struct {
StartedAt time.Time `json:"startedAt"` StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"` EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"` Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
} }
// PromptExecutionPaths records only destinations reached by a completed run. // PromptExecutionPaths records only destinations reached by a completed run.
@@ -165,7 +164,6 @@ func PromptExecutionProvenanceFrom(value promptexec.Execution) PromptExecutionPr
InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID, InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID,
ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage,
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
DataPackagePath: value.DataPackagePath,
} }
} }
@@ -255,7 +253,6 @@ func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error {
}{ }{
{"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash}, {"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash},
{"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName}, {"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName},
{"data package path", value.DataPackagePath},
} { } {
if strings.TrimSpace(required.value) == "" { if strings.TrimSpace(required.value) == "" {
return fmt.Errorf("completed prompt execution provenance %s is required", required.name) return fmt.Errorf("completed prompt execution provenance %s is required", required.name)

View File

@@ -1,207 +0,0 @@
package state
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func TestPromptPreparationArtifactValidation(t *testing.T) {
valid := validPreparationArtifact()
if err := valid.Validate(); err != nil {
t.Fatalf("valid successful preparation: %v", err)
}
failed := valid
failed.Status = PromptPreparationFailed
failed.Preparation = nil
failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
if err := failed.Validate(); err != nil {
t.Fatalf("valid failed preparation: %v", err)
}
tests := []struct {
name string
mutate func(*PromptPreparationArtifact)
}{
{"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }},
{"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }},
{"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }},
{"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }},
{"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }},
{"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }},
{"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }},
{"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }},
{"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }},
{"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
{"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }},
{"success error", func(a *PromptPreparationArtifact) {
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
}},
{"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }},
{"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }},
{"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validPreparationArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) {
tests := []struct {
name string
mutate func(*PromptPreparationArtifact)
}{
{"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }},
{"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
{"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }},
{"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }},
{"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validPreparationArtifact()
artifact.Status = PromptPreparationFailed
artifact.Preparation = nil
artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestPromptExecutionArtifactValidation(t *testing.T) {
valid := validExecutionArtifact()
valid.Provenance.RunID = "provider-run-different-from-weatherreporter"
valid.Provenance.GeneratedHash = ""
valid.Provenance.Usage = promptexec.TokenUsage{}
if err := valid.Validate(); err != nil {
t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err)
}
rejected := validExecutionArtifact()
rejected.Status = PromptExecutionValidationRejected
validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"})
rejected.Validation = &validation
if err := rejected.Validate(); err != nil {
t.Fatalf("valid validation rejection: %v", err)
}
failed := validFailedExecutionArtifact()
if err := failed.Validate(); err != nil {
t.Fatalf("valid operational failure: %v", err)
}
tests := []struct {
name string
mutate func(*PromptExecutionArtifact)
}{
{"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }},
{"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }},
{"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }},
{"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }},
{"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }},
{"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }},
{"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }},
{"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
{"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }},
{"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }},
{"wrong validation", func(a *PromptExecutionArtifact) {
value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil)
a.Validation = &value
}},
{"operational error", func(a *PromptExecutionArtifact) {
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
}},
{"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }},
{"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }},
{"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }},
{"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }},
{"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }},
{"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }},
{"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }},
{"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }},
{"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }},
{"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }},
{"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }},
{"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }},
{"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validExecutionArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) {
tests := []struct {
name string
mutate func(*PromptExecutionArtifact)
}{
{"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }},
{"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
{"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }},
{"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validFailedExecutionArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func validPreparationArtifact() PromptPreparationArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptPreparationArtifact{
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0", DataPackagePath: "/workspace/data.yaml",
Preparation: &promptexec.Preparation{
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
DataPackagePath: "/workspace/data.yaml",
},
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func validExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
Provenance: &PromptExecutionProvenance{
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
},
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func validFailedExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
}
}