eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Run report generation without workspace state

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-01 19:52:22 +00:00
parent 4bdba6f2b7
commit 7ffc3dc603
21 changed files with 343 additions and 3464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/adapters/promptkit/adapter.go
View File

@@ -124,9 +124,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
PromptID: request.PromptID,
PromptVersion: request.PromptVersion,
ProfileID: request.ProfileID,
Inputs: map[string]promptkit.ArtifactRef{
"data_package": promptkit.InlineWithURI(request.DataPackagePath, string(append([]byte(nil), request.DataPackage...))),
},
Inputs: map[string]promptkit.ArtifactRef{"data_package": promptkit.Inline(string(append([]byte(nil), request.DataPackage...)))},
})
if err != nil {
return nil, classifyError(err)
@@ -134,7 +132,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
defer prepared.Discard()
details := prepared.Details()
preparation, debug := preparationValues(details, request.DataPackagePath, request.CaptureDebug)
preparation, debug := preparationValues(details, request.CaptureDebug)
if preparedCallback != nil {
if err := preparedCallback(preparation, debug); err != nil {
return nil, err
@@ -145,7 +143,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
if err != nil {
return nil, classifyError(err)
}
return executionValue(result, request.DataPackagePath, request.CaptureDebug), nil
return executionValue(result, request.CaptureDebug), nil
}
func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
@@ -156,7 +154,7 @@ func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
}
}
func preparationValues(value promptkit.PreparedRun, dataPackagePath string, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
func preparationValues(value promptkit.PreparedRun, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
preparation := promptexec.Preparation{
PromptID: value.PromptID,
PromptVersion: value.PromptVersion,
@@ -170,7 +168,6 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: time.Duration(value.DurationMS) * time.Millisecond,
DataPackagePath: dataPackagePath,
}
if !captureDebug {
return preparation, nil
@@ -186,7 +183,7 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
return preparation, debug
}
func executionValue(value *promptkit.RunResult, dataPackagePath string, captureDebug bool) *promptexec.Execution {
func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.Execution {
if value == nil {
return nil
}
@@ -214,12 +211,11 @@ func executionValue(value *promptkit.RunResult, dataPackagePath string, captureD
CachedTokens: value.Usage.CachedTokens,
CacheWriteTokens: value.Usage.CacheWriteTokens,
},
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: value.Duration,
Validation: validation,
DataPackagePath: dataPackagePath,
RawOutput: []byte(value.RawOutput),
StartedAt: value.StartTime,
EndedAt: value.EndTime,
Duration: value.Duration,
Validation: validation,
RawOutput: []byte(value.RawOutput),
}
if captureDebug {
execution.Debug = &promptexec.ExecutionDebug{

24
internal/adapters/promptkit/adapter_test.go
View File

@@ -210,7 +210,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
callbackCalls := 0
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
callbackCalls++
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.DataPackagePath != request.DataPackagePath || preparation.ModelName != "test-model" {
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" {
t.Fatalf("preparation = %#v", preparation)
}
if debug != nil {
@@ -227,7 +227,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
if callbackCalls != 1 || client.callCount() != 1 {
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
}
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content || result.DataPackagePath != request.DataPackagePath {
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content {
t.Fatalf("result = %#v", result)
}
if result.Debug != nil {
@@ -250,11 +250,10 @@ func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
t.Fatalf("newAdapter() error = %v", err)
}
request := promptexec.ExecuteRequest{
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
DataPackagePath: "data-packages/hourly/data_package.yaml",
PromptID: "weather.hourly_generated_text",
PromptVersion: "2.0.0",
ProfileID: "weather-light",
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
}
var preparation promptexec.Preparation
prepared := false
@@ -288,7 +287,7 @@ func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
t.Fatalf("Execute() error = %v", err)
}
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != request.DataPackagePath || reader.ref.Body != string(request.DataPackage) {
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) {
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
}
}
@@ -527,11 +526,10 @@ func writeProfileFile(t *testing.T, profile string) string {
func testExecuteRequest() promptexec.ExecuteRequest {
return promptexec.ExecuteRequest{
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
DataPackagePath: "data-packages/daily/data_package.yaml",
PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0",
ProfileID: "test-profile",
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
}
}

183
internal/app/app.go
View File

@@ -21,7 +21,6 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
@@ -53,7 +52,6 @@ type GenerateRequest struct {
Collector Collector
Notifier Notifier
Executor promptexec.Executor
Store state.Store
}
type BatchRequest struct {
@@ -65,7 +63,6 @@ type BatchRequest struct {
LLMDebugDir string
Collector Collector
Executor promptexec.Executor
Store state.Store
Notifier Notifier
}
@@ -85,21 +82,22 @@ type ReportFacts struct {
}
type ReportResult struct {
ModuleSnapshot module.Snapshot
ModuleSnapshotPath string
DataPackage promptinput.Package
DataPackagePath string
PreparationPath string
ExecutionPath string
LLMDebugPath string
ReportPath string
OutputPath string
Metadata state.Metadata
MetadataPath string
GeneratedTextRawPath string
GeneratedTextPath string
RenderContextPath string
Notification *NotificationResult
ReportID report.ID
ReportName string
PromptID string
PromptVersion string
RunID string
GeneratedAt time.Time
Timezone string
ValidPeriod timeutil.Period
ProfileID string
BackendID string
ModelName string
SourceWarnings []weatherdata.SourceWarning
ValidationStatus promptexec.ValidationStatus
LLMDebugPath string
OutputPath string
Notification *NotificationResult
}
type BatchResult struct {
@@ -132,25 +130,26 @@ type BatchNotificationReport struct {
}
type BatchReportResult struct {
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
NotificationStatus string `json:"notificationStatus,omitempty"`
NotificationRunID string `json:"notificationRunId,omitempty"`
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
NotificationError string `json:"notificationError,omitempty"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
NotificationStatus string `json:"notificationStatus,omitempty"`
NotificationRunID string `json:"notificationRunId,omitempty"`
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
NotificationError string `json:"notificationError,omitempty"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
Timezone string `json:"timezone"`
ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
}
type BatchError struct {
@@ -260,14 +259,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
if err != nil {
return nil, err
}
result := initialReportResult(req, resolved, PromptInspectionResult{})
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
if err != nil {
return nil, err
return result, err
}
req.OutputPath = outputPath
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
if err != nil {
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
}
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
Resolved: resolved,
@@ -275,11 +275,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Promptkit: req.Config.Promptkit,
})
if err != nil {
return nil, err
return result, err
}
result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName
collection, err := collectWeather(ctx, req.Config, req.Collector)
if err != nil {
return nil, err
return result, err
}
return generatePromptReport(ctx, promptReportRequest{
GenerateRequest: req,
@@ -287,6 +288,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
Collection: *collection,
Inspection: inspection,
DebugWriter: debugWriter,
Result: result,
})
}
@@ -339,14 +341,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
return nil, err
}
if req.Batch == BatchEvening || req.Batch == BatchMorning {
store := req.Store
if store == nil {
defaultStore, err := defaultStore(req.Config)
if err != nil {
return nil, err
}
store = defaultStore
}
startedAt := now
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
for _, planned := range plannedReports {
@@ -362,7 +356,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
OutputPath: outputPath,
Notifier: req.Notifier,
Executor: req.Executor,
Store: store,
},
Resolved: resolved,
Collection: *collection,
@@ -398,13 +391,14 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
}
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
item.DataPackagePath = result.DataPackagePath
item.PreparationPath = result.PreparationPath
item.ExecutionPath = result.ExecutionPath
item.LLMDebugPath = result.LLMDebugPath
item.ReportPath = result.ReportPath
item.OutputPath = result.OutputPath
item.MetadataPath = result.MetadataPath
item.ProfileID = result.ProfileID
item.BackendID = result.BackendID
item.ModelName = result.ModelName
item.Timezone = result.Timezone
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
item.ValidationStatus = result.ValidationStatus
if result.Notification != nil {
item.NotificationStatus = result.Notification.Status
item.NotificationRunID = result.Notification.RunID
@@ -451,6 +445,7 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
RunID: metadata.RunID,
GeneratedAt: metadata.GeneratedAt,
ValidPeriod: metadata.ValidPeriod,
Timezone: "",
}
}
@@ -612,78 +607,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda
return bundle, nil
}
type finalizeRenderedReportRequest struct {
Config config.Config
Store state.Store
Resolved report.Resolved
Metadata state.Metadata
MetadataPath string
ExecutionArtifact *state.PromptExecutionArtifact
RenderedReportPath string
OutputPath string
Notifier Notifier
GenerationErr error
noNotify bool
}
type finalizeRenderedReportResult struct {
OutputPath string
Metadata state.Metadata
MetadataPath string
Notification *NotificationResult
}
func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) {
if req.Store == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("state store is required")
}
if req.RenderedReportPath == "" {
return finalizeRenderedReportResult{}, fmt.Errorf("rendered report path is required for report %q", req.Resolved.Definition.ID)
}
if req.ExecutionArtifact == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
}
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
if req.OutputPath != "" && req.GenerationErr == nil {
if req.OutputPath != req.RenderedReportPath {
if err := fileutil.CopyFileAtomic(req.RenderedReportPath, req.OutputPath); err != nil {
return result, err
}
result.OutputPath = req.OutputPath
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
paths.OutputPath = req.OutputPath
}); err != nil {
return result, err
}
} else {
result.OutputPath = req.OutputPath
}
}
metadata := req.Metadata
metadata.RenderedReportPath = req.RenderedReportPath
metadataPath, err := req.Store.SaveMetadata(ctx, metadata)
if err != nil {
return result, err
}
result.Metadata = metadata
result.MetadataPath = metadataPath
if req.GenerationErr != nil {
return result, req.GenerationErr
}
if req.noNotify {
return result, nil
}
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.OutputPath, metadata.RunID, metadata.GeneratedAt, req.Notifier)
result.Notification = notification
if err != nil {
return result, err
}
return result, nil
}
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
notifier, enabled := reportNotifier(cfg, notifier)
if !enabled {
@@ -965,7 +888,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected
}
}
func promptMetadata(metadata state.Metadata) promptinput.Metadata {
func promptMetadata(metadata briefing.Metadata) promptinput.Metadata {
return promptinput.Metadata{
RunID: metadata.RunID,
ReportID: metadata.ReportID,
@@ -1008,10 +931,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext {
return &location
}
func defaultStore(cfg config.Config) (*state.FilesystemStore, error) {
return state.NewFilesystemStore(cfg.Workspace)
}
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
if err == nil {
return nil

85
internal/app/batch_execution_test.go
View File

@@ -1,85 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now string
wantPrompts int
}{
{name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3},
{name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
now := mustParse(test.now)
req := BatchRequest{Config: cfg, Batch: test.batch, Now: now, WorkingDir: t.TempDir()}
candidates, err := batchInspectionCandidates(req, now)
if err != nil {
t.Fatalf("batchInspectionCandidates() error = %v", err)
}
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
"weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
}, prompts: map[string]promptexec.PromptInspection{}}
for _, candidate := range candidates {
executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition)
}
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
return nil, errors.New("collection reached")
})
req.Executor = executor
req.Collector = collector
_, err = RunBatchDetailed(context.Background(), req)
if err == nil || err.Error() != "collection reached" {
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
}
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
}
})
}
}
func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) {
item := BatchReportResult{}
copyBatchReportPaths(&item, &ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
})
data, err := json.Marshal(item)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("batch item omits reached paths:\n%s", text)
}
}
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
return f(ctx, req)
}
var _ Collector = collectorFunc(nil)

449
internal/app/batch_workflow_test.go
View File

@@ -1,449 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type assembledBatchExecutor struct {
definitions map[string]report.Definition
promptRequests []string
profileRequests []string
executeRequests []promptexec.ExecuteRequest
failures map[int]error
active int
maxActive int
}
func newAssembledBatchExecutor() *assembledBatchExecutor {
definitions := make(map[string]report.Definition)
for _, definition := range report.DefaultRegistry().All() {
definitions[definition.PromptID] = definition
}
return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)}
}
func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
e.promptRequests = append(e.promptRequests, id+"@"+version)
definition, ok := e.definitions[id]
if !ok || definition.PromptVersion != version {
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
}
return logicalPromptInspection(definition), nil
}
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
e.profileRequests = append(e.profileRequests, id)
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
call := len(e.executeRequests)
e.executeRequests = append(e.executeRequests, req)
e.active++
if e.active > e.maxActive {
e.maxActive = e.active
}
defer func() { e.active-- }()
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if err := e.failures[call]; err != nil {
return nil, err
}
definition := e.definitions[req.PromptID]
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(generatedTextForPrompt(req.PromptID)),
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
type assembledBatchNotifier struct {
reportRequests []NotificationRequest
batchRequests []batchNotificationRequest
batchResult *NotificationResult
batchErr error
}
func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.reportRequests = append(n.reportRequests, req)
return nil, errors.New("per-report notification must be suppressed")
}
func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) {
n.batchRequests = append(n.batchRequests, req)
if n.batchErr != nil {
return nil, n.batchErr
}
if n.batchResult != nil {
result := *n.batchResult
if result.PipelineID == "" {
result.PipelineID = req.PipelineID
}
if result.BundleID == "" {
result.BundleID = req.BundleID
}
if result.IdempotencyKey == "" {
result.IdempotencyKey = req.IdempotencyKey
}
return &result, nil
}
return &NotificationResult{
RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
tests := []struct {
name string
batch BatchKind
now time.Time
wantIDs []report.ID
wantCopies []string
}{
{name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}},
{name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: test.batch, Now: test.now, WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: collector, Executor: executor,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 {
t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed)
}
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
}
if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests)
}
for index, item := range result.Reports {
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {
t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index])
}
if executor.executeRequests[index].PromptID != item.PromptID {
t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" {
t.Fatalf("successful report paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
if filepath.Base(item.OutputPath) != test.wantCopies[index] {
t.Fatalf("output = %q, want %q", item.OutputPath, test.wantCopies[index])
}
assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath)
renderedReport, readErr := os.ReadFile(item.ReportPath)
if readErr != nil {
t.Fatalf("read rendered report: %v", readErr)
}
copied, readErr := os.ReadFile(item.OutputPath)
if readErr != nil || !bytes.Equal(renderedReport, copied) {
t.Fatalf("output mismatch/error = %v", readErr)
}
}
})
}
}
func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := newAssembledBatchExecutor()
executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(),
OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"),
Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed)
}
if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" {
t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification)
}
for index, item := range result.Reports {
if index == 1 {
if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) {
t.Fatalf("failed item = %#v", item)
}
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" {
t.Fatalf("failed item reached paths = %#v", item)
}
assertBatchItemMatchesMetadata(t, item)
continue
}
if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" {
t.Fatalf("continued item %d = %#v", index, item)
}
}
}
func TestRunBatchDetailedNotificationLifecycle(t *testing.T) {
t.Run("disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("batch disabled", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
cfg.Notify.Distributor.Batch.Enabled = false
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
}
})
t.Run("all success", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted",
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
outputDir := filepath.Join(t.TempDir(), "output")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 {
t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests))
}
outputPaths := make(map[string]struct{}, len(result.Reports))
for _, item := range result.Reports {
outputPaths[item.OutputPath] = struct{}{}
}
request := notifier.batchRequests[0]
if len(request.IncludedReports) != len(result.Reports) {
t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports))
}
for _, file := range request.Files {
if _, ok := outputPaths[file.SourcePath]; !ok || !strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" {
t.Fatalf("notification file = %#v, want selected Markdown output source", file)
}
}
})
t.Run("upload failure", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil {
t.Fatalf("RunBatchDetailed() error = %v", err)
}
if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" {
t.Fatalf("result = %#v, want successful reports and failed notification", result)
}
for _, item := range result.Reports {
if item.Status != "succeeded" {
t.Fatalf("report item = %#v, want success despite notification failure", item)
}
}
if !strings.Contains(result.Notification.Error, "batch upload rejected") {
t.Fatalf("notification error = %#v", result.Notification)
}
})
t.Run("status report", func(t *testing.T) {
cfg := assembledBatchConfig(t, true)
bundle := assembledBatchBundle(t, "2026-05-31")
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted",
StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
}}
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
})
if err != nil || result.Failed != 0 || result.Notification == nil {
t.Fatalf("result/error = %#v/%v", result, err)
}
if result.Notification.Status != "accepted" || result.Notification.RunID != "batch-notification-run" {
t.Fatalf("notification = %#v", result.Notification)
}
})
}
func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) {
cfg := assembledBatchConfig(t, false)
bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01")
outputDir := filepath.Join(t.TempDir(), "output")
debugRoot := filepath.Join(t.TempDir(), "debug")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(),
})
if err != nil || result.Failed != 0 || len(result.Reports) != 3 {
t.Fatalf("result/error = %#v/%v", result, err)
}
seenRuns := make(map[string]struct{})
seenDebug := make(map[string]struct{})
dailyDates := make(map[string]BatchReportResult)
for _, item := range result.Reports {
if _, exists := seenRuns[item.RunID]; exists {
t.Fatalf("duplicate run ID %q", item.RunID)
}
seenRuns[item.RunID] = struct{}{}
if _, exists := seenDebug[item.LLMDebugPath]; exists {
t.Fatalf("duplicate debug path %q", item.LLMDebugPath)
}
seenDebug[item.LLMDebugPath] = struct{}{}
if item.ReportID == report.Daily {
date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02")
dailyDates[date] = item
}
}
for _, date := range []string{"2026-05-31", "2026-06-01"} {
item, ok := dailyDates[date]
if !ok {
t.Fatalf("daily items = %#v, want %s", dailyDates, date)
}
if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") {
t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath)
}
wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID)
if item.LLMDebugPath != wantDebugPrefix {
t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix)
}
assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json"))
}
}
func assembledBatchConfig(t *testing.T, notify bool) config.Config {
t.Helper()
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = notify
cfg.Notify.Distributor.Batch.Enabled = notify
return cfg
}
func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle {
t.Helper()
bundle := workflowBundle(t)
location := mustLoadTestLocation(t, "America/Chicago")
for _, date := range dates {
periods := fullDayPeriods(t, date, location)
for index := range periods {
temperature := float64(60 + index)
periods[index].TemperatureF = &temperature
periods[index].TextDescription = "Partly cloudy"
}
bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...)
}
return bundle
}
func generatedTextForPrompt(promptID string) string {
switch promptID {
case "weather.today_generated_text":
return validTodayWorkflowJSON()
case "weather.tomorrow_generated_text":
return validTomorrowWorkflowJSON()
case "weather.daily_generated_text":
return validDailyWorkflowJSON()
case "weather.hourly_generated_text":
return validHourlyWorkflowJSON()
default:
return ""
}
}
func assertBatchPathsExist(t *testing.T, paths ...string) {
t.Helper()
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected path %q: %v", path, err)
}
}
}
func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) {
t.Helper()
data, err := os.ReadFile(item.MetadataPath)
if err != nil {
t.Fatalf("read metadata %q: %v", item.MetadataPath, err)
}
var metadata state.Metadata
if err := json.Unmarshal(data, &metadata); err != nil {
t.Fatalf("decode metadata %q: %v", item.MetadataPath, err)
}
if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID ||
item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath ||
item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath {
t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata)
}
}
func loadBatchDataPackage(t *testing.T, path string) promptinput.Package {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read data package: %v", err)
}
pkg, err := promptinput.LoadYAML(data)
if err != nil {
t.Fatalf("LoadYAML() error = %v", err)
}
return pkg
}

103
internal/app/generation_test.go Normal file
View File

@@ -0,0 +1,103 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type generationCollector struct {
bundle *weatherdata.Bundle
err error
}
func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
return &collect.Result{Bundle: c.bundle}, c.err
}
type generationExecutor struct{ called bool }
func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
definition := report.DefaultRegistry().MustLookup(report.Daily)
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
}
func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
}
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
return nil, err
}
e.called = true
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil
}
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
bundle := generationBundle(t)
executor := &generationExecutor{}
workingDir := t.TempDir()
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
t.Fatalf("result = %#v", result)
}
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
t.Fatalf("workspace state = %v, want absent", err)
}
data, err := os.ReadFile(result.OutputPath)
if err != nil || len(data) == 0 {
t.Fatalf("output = %q, error = %v", data, err)
}
}
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
collectionErr := errors.New("weather source unavailable")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{},
})
if !errors.Is(err, collectionErr) {
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
}
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
t.Fatalf("result = %#v", result)
}
}
func generationBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
return bundle
}
func generationTime(value string) time.Time {
parsed, _ := time.Parse(time.RFC3339, value)
return parsed
}
var _ promptexec.Executor = (*generationExecutor)(nil)
var _ Collector = generationCollector{}
var _ = report.Daily

4
internal/app/inspect.go
View File

@@ -33,7 +33,7 @@ type SourceInspection struct {
}
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
store, err := defaultStore(req.Config)
store, err := state.NewFilesystemStore(req.Config.Workspace)
if err != nil {
return nil, err
}
@@ -94,7 +94,7 @@ type runInspection struct {
}
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
store, err := defaultStore(req.Config)
store, err := state.NewFilesystemStore(req.Config.Workspace)
if err != nil {
return runInspection{}, err
}

129
internal/app/output_test.go
View File

@@ -1,129 +0,0 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func TestResolveReportOutputPath(t *testing.T) {
workingDir := t.TempDir()
cfg := config.Defaults()
now := workflowTime("2026-05-29T08:30:00-05:00")
for _, test := range []struct {
report ReportKind
date string
name string
}{
{report: ReportDaily, date: "2026-05-30T12:00:00-05:00", name: "daily-2026-05-30.md"},
{report: ReportToday, date: "2026-05-29T12:00:00-05:00", name: "today.md"},
{report: ReportTomorrow, name: "tomorrow.md"},
{report: ReportHourly, name: "hourly.md"},
} {
t.Run(string(test.report), func(t *testing.T) {
req := GenerateRequest{Config: cfg, Report: test.report}
if test.date != "" {
req.Date = workflowTime(test.date)
}
resolved, err := ResolveGenerate(req, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
path, err := resolveReportOutputPath(workingDir, "", resolved)
if err != nil {
t.Fatalf("resolveReportOutputPath() error = %v", err)
}
if path != filepath.Join(workingDir, test.name) {
t.Fatalf("path = %q, want %q", path, filepath.Join(workingDir, test.name))
}
})
}
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-30T12:00:00-05:00"),
}, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
absoluteDir := t.TempDir()
for _, test := range []struct {
override string
want string
}{
{override: filepath.Join("reports", "custom.md"), want: filepath.Join(workingDir, "reports", "custom.md")},
{override: filepath.Join(absoluteDir, "custom.md"), want: filepath.Join(absoluteDir, "custom.md")},
} {
path, err := resolveReportOutputPath(workingDir, test.override, resolved)
if err != nil {
t.Fatalf("resolveReportOutputPath(%q) error = %v", test.override, err)
}
if path != filepath.Clean(test.want) {
t.Fatalf("path = %q, want %q", path, filepath.Clean(test.want))
}
}
}
func TestGenerateDetailedRejectsInvalidOutputBeforeCollection(t *testing.T) {
cfg := workflowConfig(t)
collector := &workflowCollector{err: errors.New("collection must not run")}
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: t.TempDir(),
Collector: collector,
})
if err == nil || collector.calls != 0 {
t.Fatalf("GenerateDetailed() error/calls = %v/%d, want invalid output before collection", err, collector.calls)
}
}
func TestGenerateDetailedPreservesExistingOutputWhenGenerationFails(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
workingDir := t.TempDir()
outputPath := filepath.Join(workingDir, "daily-2026-05-29.md")
if err := os.WriteFile(outputPath, []byte("existing report"), 0o600); err != nil {
t.Fatalf("write existing output: %v", err)
}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: &workflowExecutor{definition: report.DefaultRegistry().MustLookup(report.Daily), raw: []byte(`{}`)},
})
if err == nil || result == nil || result.OutputPath != "" {
t.Fatalf("result/error/output = %#v/%v/%q, want failed generation without output publication", result, err, result.OutputPath)
}
data, readErr := os.ReadFile(outputPath)
if readErr != nil || string(data) != "existing report" {
t.Fatalf("output after failure = %q, error %v, want preserved content", data, readErr)
}
}
func TestRunBatchDetailedUsesWorkingDirectoryForOutput(t *testing.T) {
cfg := assembledBatchConfig(t, false)
workingDir := t.TempDir()
bundle := assembledBatchBundle(t, "2026-05-31")
result, err := RunBatchDetailed(context.Background(), BatchRequest{
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: workingDir,
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: newAssembledBatchExecutor(),
})
if err != nil || result == nil || result.Failed != 0 {
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
}
if len(result.Reports) != 2 {
t.Fatalf("reports = %#v, want Tomorrow and Daily", result.Reports)
}
if result.Reports[0].OutputPath != filepath.Join(workingDir, "tomorrow.md") ||
result.Reports[1].OutputPath != filepath.Join(workingDir, "daily-2026-05-31.md") {
t.Fatalf("output paths = %#v, want working-directory defaults", result.Reports)
}
}

523
internal/app/prompt_artifact_paths_test.go
View File

@@ -1,523 +0,0 @@
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failGeneratedText = "generated text"
failRenderContext = "render context"
failRenderedReportPath = "rendered report path"
)
type failingPersistenceStore struct {
state.Store
failOperation string
failExecutionCall int
failMetadataCall int
executionCalls int
metadataCalls int
renderedReportPath string
}
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
s.executionCalls++
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
return "", errors.New("injected prompt execution persistence failure")
}
return s.Store.SavePromptExecution(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
if s.failOperation == failGeneratedText {
return "", errors.New("injected generated text persistence failure")
}
return s.Store.SaveGeneratedText(ctx, resolved, data)
}
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
if s.failOperation == failRenderContext {
return "", errors.New("injected render context persistence failure")
}
return s.Store.SaveRenderContext(ctx, resolved, value)
}
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
if s.failOperation == failRenderedReportPath {
return s.renderedReportPath, nil
}
return s.Store.PrepareRenderedReport(ctx, resolved)
}
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
s.metadataCalls++
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
return "", errors.New("injected metadata persistence failure")
}
return s.Store.SaveMetadata(ctx, metadata)
}
type artifactPathExecutor struct {
beforePreparationErr error
afterPreparationErr error
validation promptexec.ValidationStatus
}
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
}
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
if err := callback(promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
}, nil); err != nil {
return nil, err
}
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`),
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
}, nil
}
type successfulNotifier struct{}
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
}
type failingNotifier struct{}
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return nil, errors.New("injected notification failure")
}
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
tests := []struct {
name string
failOperation string
failMetadataCall int
output bool
notify bool
want reachedPromptArtifacts
}{
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
{name: "rendered report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
{name: "output then metadata", failOperation: failMetadata, failMetadataCall: 5, output: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
req.Store = store
if test.output {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
})
}
}
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
tests := []struct {
name string
executor artifactPathExecutor
want reachedPromptArtifacts
wantExecutionStatus state.PromptExecutionStatus
wantExecutionPaths state.PromptExecutionPaths
wantRawExecution bool
}{
{
name: "preparation failure",
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
want: reachedPromptArtifacts{preparation: true, metadata: true},
},
{
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
wantExecutionStatus: state.PromptExecutionFailed,
},
{
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
wantExecutionStatus: state.PromptExecutionValidationRejected,
wantRawExecution: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, test.executor)
if test.wantRawExecution {
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
if test.wantExecutionStatus != "" {
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if loadErr != nil {
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
}
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
}
}
})
}
}
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
result, err := generatePromptReport(context.Background(), req)
if err != nil {
t.Fatalf("generatePromptReport() error = %v", err)
}
want := state.PromptExecutionPaths{
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
OutputPath: paths.output,
}
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
data, err := os.ReadFile(result.ExecutionPath)
if err != nil {
t.Fatalf("read execution artifact: %v", err)
}
text := string(data)
for _, forbidden := range []string{
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
"credential", "secret-value",
} {
if strings.Contains(text, forbidden) {
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
}
}
}
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
tests := []struct {
name string
failOperation string
failExecutionCall int
failMetadataCall int
requestOutput bool
failOutput bool
notify bool
notificationFailure bool
wantExecution reachedExecutionArtifacts
wantResult reachedPromptArtifacts
}{
{
name: "normalized text write", failOperation: failGeneratedText,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
},
{
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context write", failOperation: failRenderContext,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "rendered report write", failOperation: failRenderedReportPath,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "rendered report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output write", requestOutput: true, failOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "output metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{
Store: req.Store, failOperation: test.failOperation,
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
}
if test.failOperation == failRenderedReportPath {
store.renderedReportPath = t.TempDir()
}
req.Store = store
if test.requestOutput {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.failOutput {
blocker := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
t.Fatalf("write output blocker: %v", err)
}
req.OutputPath = filepath.Join(blocker, "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
if test.notificationFailure {
req.Notifier = failingNotifier{}
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
})
}
}
type reachedExecutionArtifacts struct {
raw bool
normalized bool
renderContext bool
report bool
output bool
}
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
result := state.PromptExecutionPaths{}
if reached.raw {
result.RawOutputPath = paths.GeneratedTextRaw
}
if reached.normalized {
result.GeneratedTextPath = paths.GeneratedText
}
if reached.renderContext {
result.RenderContextPath = paths.RenderContext
}
if reached.report {
result.RenderedReportPath = paths.RenderedReport
}
if reached.output {
result.OutputPath = paths.output
}
return result
}
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
t.Helper()
artifact, err := store.LoadPromptExecution(context.Background(), path)
if err != nil {
t.Fatalf("LoadPromptExecution() error = %v", err)
}
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
}
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
}
if artifact.Paths != want {
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
}
}
type promptArtifactPaths struct {
state.ArtifactPaths
output string
}
type reachedPromptArtifacts struct {
preparation bool
execution bool
metadata bool
raw bool
normalized bool
renderContext bool
report bool
output bool
}
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
}, mustParse("2026-05-29T05:00:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read daily fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(bundleData, &bundle); err != nil {
t.Fatalf("decode daily fixture: %v", err)
}
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystemStore.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
debugWriter, err := promptdebug.NewPromptDebugWriter("")
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
return promptReportRequest{
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
Inspection: PromptInspectionResult{
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
},
DebugWriter: debugWriter, noNotify: true,
}, promptArtifactPaths{ArtifactPaths: paths}
}
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
t.Helper()
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
}
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
}
checks := []struct {
name string
got string
metadataGot string
inMetadata bool
path string
want bool
}{
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
{"output", result.OutputPath, "", false, paths.output, want.output},
}
for _, check := range checks {
if check.want && check.got != check.path {
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
}
if check.want && check.inMetadata && check.metadataGot != check.path {
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
}
if !check.want && check.got != "" {
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
}
if !check.want && check.inMetadata && check.metadataGot != "" {
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
}
}
}

379
internal/app/prompt_generate.go
View File

@@ -3,7 +3,6 @@ package app
import (
"context"
"fmt"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
@@ -14,7 +13,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type promptReportRequest struct {
@@ -23,9 +22,23 @@ type promptReportRequest struct {
Collection collect.Result
Inspection PromptInspectionResult
DebugWriter *promptdebug.PromptDebugWriter
Result *ReportResult
noNotify bool
}
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
result *ReportResult
briefingMetadata briefing.Metadata
reportFacts ReportFacts
moduleSnapshot module.Snapshot
dataPackage []byte
handler generatedtext.Handler
debugRef promptdebug.PromptDebugRef
callbackFailed bool
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
workflow, err := newPromptReportWorkflow(ctx, req)
if err != nil {
@@ -34,369 +47,145 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
if err := workflow.buildInputs(); err != nil {
return workflow.result, err
}
execution, executeErr := workflow.executePrompt()
if executeErr != nil {
return workflow.result, workflow.handleExecutionFailure(executeErr)
execution, err := workflow.executePrompt()
if err != nil {
if workflow.callbackFailed {
return workflow.result, err
}
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
}
if execution == nil {
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
}
return workflow.result, workflow.reportError("execute prompt", err)
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
}
if err := workflow.persistExecutionDebug(*execution); err != nil {
workflow.result.ValidationStatus = execution.Validation.Status
if err := workflow.writeExecutionDebug(*execution); err != nil {
return workflow.result, err
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
return workflow.result, saveErr
}
return workflow.result, workflow.reportError("validate prompt execution", err)
}
if err := workflow.persistCompletedExecution(*execution); err != nil {
return workflow.result, err
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
}
if execution.Validation.Status == promptexec.ValidationFailed {
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)
return workflow.result, workflow.reportError("validate prompt execution", err)
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
}
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
if err != nil {
return workflow.result, err
}
return workflow.finalizeReport(rendered)
}
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
store state.Store
result *ReportResult
metadata state.Metadata
briefingMetadata briefing.Metadata
reportFacts ReportFacts
moduleSnapshot module.Snapshot
dataPackageBytes []byte
handler generatedtext.Handler
executionArtifact state.PromptExecutionArtifact
debugRef promptdebug.PromptDebugRef
prepared bool
callbackFailed bool
return workflow.renderAndPublish(execution.RawOutput)
}
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
if req.Collection.Bundle == nil {
return nil, fmt.Errorf("collected weather bundle is required")
}
store := req.Store
var err error
if store == nil {
store, err = defaultStore(req.Config)
if err != nil {
return nil, err
}
result := req.Result
if result == nil {
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
}
return &promptReportWorkflow{
ctx: ctx, req: req,
result: result,
}, nil
}
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
metadata := resolved.Metadata()
return &ReportResult{
ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name,
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone,
ValidPeriod: metadata.ValidPeriod,
ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName,
}
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
}
func (w *promptReportWorkflow) buildInputs() error {
paths, err := w.store.Paths(w.req.Resolved)
if err != nil {
return err
}
w.result = &ReportResult{}
var err error
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err)
return w.reportError("build report facts", err)
}
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
if err != nil {
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err)
return w.reportError("build module snapshot", err)
}
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
if err != nil {
return err
}
w.result.ModuleSnapshot = w.moduleSnapshot
w.result.ModuleSnapshotPath = moduleSnapshotPath
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{
ModuleSnapshot: moduleSnapshotPath,
Metadata: paths.Metadata,
})
w.result.Metadata = w.metadata
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot,
})
w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
if err != nil {
return w.reportError("build data package", err)
}
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage)
w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
if err != nil {
return err
return w.reportError("marshal data package", err)
}
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
if err != nil {
return err
}
w.metadata.DataPackagePath = dataPackagePath
w.result.DataPackage = dataPackage
w.result.DataPackagePath = dataPackagePath
w.result.Metadata = w.metadata
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
if err != nil {
return w.reportError("lookup generated text catalog", err)
}
w.debugRef = promptdebug.PromptDebugRef{
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
}
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
return nil
}
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
}, w.persistPreparation)
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
}
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
if w.req.DebugWriter == nil {
return nil
}
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
if err != nil {
w.callbackFailed = true
return err
}
w.prepared = true
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
if err != nil {
w.callbackFailed = true
return promptDebugWriteError(err)
}
if debugPath != "" {
w.result.LLMDebugPath = debugPath
}
if err := w.saveMetadata(); err != nil {
w.callbackFailed = true
return err
}
w.result.LLMDebugPath = path
return nil
}
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error {
if w.callbackFailed {
return executeErr
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
if w.req.DebugWriter == nil {
return nil
}
executeErr = classifiedPromptError("prompt execution failed", executeErr)
if !w.prepared {
if err := w.persistPreparationFailure(executeErr); err != nil {
return err
}
return w.reportError("prepare prompt", executeErr)
}
if promptexec.CategoryOf(executeErr) != "" {
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
return err
}
}
return w.reportError("execute prompt", executeErr)
}
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
startedAt, endedAt := time.Now(), time.Now()
artifact := state.PromptPreparationArtifact{
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
Error: state.NewPromptArtifactError(executeErr),
}
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.PreparationPath = path
w.metadata.PreparationPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
if err != nil {
return err
}
w.result.ExecutionPath = path
w.metadata.ExecutionPath = path
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
if err != nil {
return w.reportError("write prompt debug", promptDebugWriteError(err))
}
if debugPath != "" {
w.result.LLMDebugPath = debugPath
if path != "" {
w.result.LLMDebugPath = path
}
return nil
}
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error {
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput)
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
generatedText, _, err := w.handler.Validate(raw)
if err != nil {
return err
return w.result, w.reportError("validate generated text", err)
}
w.result.GeneratedTextRawPath = rawPath
w.metadata.GeneratedTextRawPath = rawPath
w.result.Metadata = w.metadata
w.executionArtifact = state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion,
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
}
if execution.Validation.Status == promptexec.ValidationPassed {
w.executionArtifact.Status = state.PromptExecutionSucceeded
} else {
w.executionArtifact.Status = state.PromptExecutionValidationRejected
}
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
if err != nil {
return err
}
w.result.ExecutionPath = executionPath
w.metadata.ExecutionPath = executionPath
w.result.Metadata = w.metadata
return w.saveMetadata()
}
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
generatedText, normalized, err := w.handler.Validate(raw)
if err != nil {
return nil, w.reportError("validate generated text", err)
}
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
if err != nil {
return nil, err
}
w.result.GeneratedTextPath = generatedTextPath
w.metadata.GeneratedTextPath = generatedTextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
return nil, err
}
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
if err != nil {
return nil, w.reportError("build render context", err)
return w.result, w.reportError("build render context", err)
}
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
if err != nil {
return nil, err
}
w.result.RenderContextPath = renderContextPath
w.metadata.RenderContextPath = renderContextPath
w.result.Metadata = w.metadata
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
return nil, err
}
rendered, err := w.handler.Render(renderContext)
if err != nil {
return nil, w.reportError("render template", err)
return w.result, w.reportError("render template", err)
}
return rendered, nil
}
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) {
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved)
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
return w.result, err
}
w.result.OutputPath = w.req.OutputPath
if w.req.noNotify {
return w.result, nil
}
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
w.result.Notification = notification
if err != nil {
return w.result, err
}
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
return w.result, err
}
w.result.ReportPath = reportPath
w.metadata.RenderedReportPath = reportPath
w.result.Metadata = w.metadata
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
return w.result, err
}
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
ExecutionArtifact: &w.executionArtifact, RenderedReportPath: reportPath, OutputPath: w.req.OutputPath,
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
})
w.result.OutputPath = finalized.OutputPath
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
return w.result, err
}
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
}
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
if err := w.persistReachedPath(update); err != nil {
return err
}
return w.saveMetadata()
}
func (w *promptReportWorkflow) saveMetadata() error {
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
if err != nil {
return err
}
w.result.Metadata = w.metadata
w.result.MetadataPath = path
return nil
return w.result, nil
}
func (w *promptReportWorkflow) reportError(operation string, err error) error {
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err)
}
func persistReachedPromptPath(
ctx context.Context,
store state.Store,
resolved report.Resolved,
artifact *state.PromptExecutionArtifact,
update func(*state.PromptExecutionPaths),
) error {
update(&artifact.Paths)
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
return err
}
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
now := time.Now()
return state.PromptExecutionArtifact{
SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed,
ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now,
Error: state.NewPromptArtifactError(err),
}
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
}
func classifiedPromptError(operation string, err error) error {
@@ -409,5 +198,3 @@ func classifiedPromptError(operation string, err error) error {
func promptDebugWriteError(err error) error {
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
}
func ptr[T any](value T) *T { return &value }

671
internal/app/single_report_workflow_test.go
View File

@@ -1,671 +0,0 @@
package app
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
type workflowCollector struct {
result *collect.Result
err error
calls int
}
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
c.calls++
if c.err != nil {
return nil, c.err
}
return c.result, nil
}
type workflowExecutor struct {
definition report.Definition
raw []byte
prompt promptexec.PromptInspection
inspectionErr error
profile promptexec.ProfileInspection
profileErr error
beforePreparationErr error
afterCallbackErr error
afterPreparationErr error
validation promptexec.ValidationStatus
executeCalls int
providerCalls int
request promptexec.ExecuteRequest
beforeProvider func()
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
preparation *promptexec.Preparation
execution *promptexec.Execution
}
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
if e.inspectionErr != nil {
return promptexec.PromptInspection{}, e.inspectionErr
}
if id != e.definition.PromptID || version != e.definition.PromptVersion {
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
}
if e.prompt.PromptID != "" {
return e.prompt, nil
}
return validPromptInspection(e.definition), nil
}
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
if e.profileErr != nil {
return promptexec.ProfileInspection{}, e.profileErr
}
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
}
return profile, nil
}
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
e.executeCalls++
e.request = req
if e.beforePreparationErr != nil {
return nil, e.beforePreparationErr
}
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"}
}
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
e.preparation = &preparation
if err := callback(preparation, e.preparationDebug); err != nil {
return nil, err
}
if e.afterCallbackErr != nil {
return nil, e.afterCallbackErr
}
if e.beforeProvider != nil {
e.beforeProvider()
}
e.providerCalls++
if e.afterPreparationErr != nil {
return nil, e.afterPreparationErr
}
validation := e.validation
if validation == "" {
validation = promptexec.ValidationPassed
}
execution := &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
Debug: e.executionDebug,
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}
e.execution = execution
return execution, nil
}
type workflowNotifier struct {
requests []NotificationRequest
err error
}
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
n.requests = append(n.requests, req)
if n.err != nil {
return nil, n.err
}
return &NotificationResult{
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
}, nil
}
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
date time.Time
raw string
wantOutput string
}{
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(test.id)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
outputPath := filepath.Join(t.TempDir(), test.name+".md")
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
}
if executor.request.PromptVersion != definition.PromptVersion {
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
}
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
if storeErr != nil {
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
}
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
}
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
}
persisted, readErr := os.ReadFile(result.DataPackagePath)
if readErr != nil {
t.Fatalf("read data package: %v", readErr)
}
if !bytes.Equal(executor.request.DataPackage, persisted) {
t.Fatal("executor data package differs from exact persisted YAML bytes")
}
renderedReport, readErr := os.ReadFile(result.ReportPath)
if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) {
t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput)
}
copied, readErr := os.ReadFile(outputPath)
if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath {
t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath)
}
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath {
t.Fatalf("notification requests = %#v, want selected output source", notifier.requests)
}
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
if notifier.requests[0].PipelineID != wantPipeline {
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
}
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
}
renderedName := filepath.Base(result.ReportPath)
if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath)
}
})
}
}
func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
raw string
override string
profile promptexec.ProfileInspection
}{
{
name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"},
},
{
name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
},
{
name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile",
profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Promptkit.Profile = test.override
definition := report.DefaultRegistry().MustLookup(test.id)
executor := &workflowExecutor{
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
}
bundle := workflowBundle(t)
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if executor.request.ProfileID != test.profile.ProfileID {
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
}
if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName {
t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName {
t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
})
}
}
type preparationFailingStore struct {
state.Store
}
type renderContextFailingStore struct {
state.Store
}
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
if err != nil {
return "", err
}
for index := range snapshot.Outputs {
snapshot.Outputs[index].Value = "invalid module value"
}
return path, nil
}
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
return "", errors.New("injected preparation persistence failure")
}
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
wantPreparation bool
wantExecution bool
wantRaw bool
wantProviderCall int
}{
{name: "preparation", configure: func(e *workflowExecutor) {
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true},
{name: "credential disappears", configure: func(e *workflowExecutor) {
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "canceled", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "deadline", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "generation", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation error", configure: func(e *workflowExecutor) {
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
test.configure(executor)
bundle := workflowBundle(t)
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: collector, Executor: executor,
})
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
}
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
}
})
}
}
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
tests := []struct {
name string
configure func(*workflowExecutor)
wantCategory promptexec.ErrorCategory
}{
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
{name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration},
{name: "malformed profile", configure: func(e *workflowExecutor) {
e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret")
}, wantCategory: promptexec.InvalidConfiguration},
{name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration},
{name: "credential", configure: func(e *workflowExecutor) {
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
}, wantCategory: promptexec.MissingCredential},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition}
test.configure(executor)
collector := &workflowCollector{err: errors.New("collector must not run")}
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: collector, Executor: executor,
})
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
}
if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") {
t.Fatalf("error leaks profile details: %v", err)
}
entries, readErr := os.ReadDir(cfg.Workspace.Root)
if readErr != nil || len(entries) != 0 {
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
}
})
}
}
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
})
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
}
}
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
paths, err := filesystem.Paths(resolved)
if err != nil {
t.Fatalf("Paths() error = %v", err)
}
checked := false
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
executor.beforeProvider = func() {
checked = true
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
}
}
bundle := workflowBundle(t)
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.Store = filesystem
result, err := GenerateDetailed(context.Background(), request)
if err != nil || result == nil || !checked || result.OutputPath == "" {
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
}
}
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
tests := []struct {
name string
raw string
configure func(*GenerateRequest, *workflowNotifier)
wantRaw bool
wantNormalized bool
wantContext bool
wantReport bool
wantOutput bool
}{
{name: "generated text decode", raw: `{`, wantRaw: true},
{name: "generated text domain", raw: `{}`, wantRaw: true},
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = renderContextFailingStore{Store: req.Store}
}, wantRaw: true, wantNormalized: true},
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
}, wantRaw: true, wantNormalized: true},
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
blocker := filepath.Join(t.TempDir(), "report-blocker")
if err := os.Mkdir(blocker, 0o700); err != nil {
t.Fatalf("create report blocker: %v", err)
}
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
}, wantRaw: true, wantNormalized: true, wantContext: true},
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
notifier.err = errors.New("notification rejected")
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
definition := report.DefaultRegistry().MustLookup(report.Daily)
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
notifier := &workflowNotifier{}
bundle := workflowBundle(t)
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
req := GenerateRequest{
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
Executor: executor, Notifier: notifier, Store: filesystem,
}
if test.configure != nil {
test.configure(&req, notifier)
}
result, err := GenerateDetailed(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
}
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
(result.OutputPath != "") != test.wantOutput {
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath)
}
if test.wantRaw {
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
}
}
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) {
t.Fatalf("notification requests = %#v", notifier.requests)
}
if test.name == "notification" && result.Metadata.NotificationPath != "" {
t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata)
}
})
}
}
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
tests := []struct {
name string
createCollision func(string, report.Resolved) error
wantProviderCalls int
wantPreparationFile bool
}{
{
name: "preparation debug",
createCollision: func(root string, resolved report.Resolved) error {
path := workflowDebugRunPath(root, resolved)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
return os.WriteFile(path, []byte("not a directory"), 0o600)
},
},
{
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
createCollision: func(root string, resolved report.Resolved) error {
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Notify.Distributor.Enabled = false
now := workflowTime("2026-05-29T08:30:00-05:00")
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
resolved, err := ResolveGenerate(request, now)
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
if err := test.createCollision(debugRoot, resolved); err != nil {
t.Fatalf("create debug collision: %v", err)
}
bundle := workflowBundle(t)
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
request.Executor = executor
request.LLMDebugDir = debugRoot
result, err := GenerateDetailed(context.Background(), request)
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
}
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
}
if test.wantPreparationFile && result.MetadataPath == "" {
t.Fatal("execution debug failure lost previously persisted metadata")
}
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
_, statErr := os.Stat(preparationDebug)
if (statErr == nil) != test.wantPreparationFile {
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
}
})
}
}
func workflowDebugRunPath(root string, resolved report.Resolved) string {
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
}
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
switch id {
case report.Daily:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
case report.Today:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
case report.Tomorrow:
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
case report.Hourly:
return []string{"hourly/index.md"}
default:
return nil
}
}
func workflowConfig(t *testing.T) config.Config {
t.Helper()
cfg := config.Defaults()
cfg.Workspace.Root = t.TempDir()
cfg.WeatherAPI.Timezone = "America/Chicago"
cfg.Location.ID = "home"
cfg.Location.Name = "Testville"
cfg.Location.Region = "MO"
cfg.Notify.Distributor.Enabled = true
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
return cfg
}
func workflowBundle(t *testing.T) weatherdata.Bundle {
t.Helper()
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
if err != nil {
t.Fatalf("read bundle fixture: %v", err)
}
var bundle weatherdata.Bundle
if err := json.Unmarshal(data, &bundle); err != nil {
t.Fatalf("decode bundle fixture: %v", err)
}
future := bundle.Hourly.Periods[0]
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
futureNarrative := bundle.Narrative.Periods[0]
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
futureNarrative.Name = "Tomorrow"
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
return bundle
}
func workflowTime(value string) time.Time {
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
panic(err)
}
return parsed
}
func validHourlyWorkflowJSON() string {
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`
}
func validTomorrowWorkflowJSON() string {
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validTodayWorkflowJSON() string {
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
}
func validDailyWorkflowJSON() string {
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
}

66
internal/cli/result.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
)
const (
@@ -17,26 +18,25 @@ const (
)
type generateSummary struct {
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
ReportPath string `json:"reportPath,omitempty"`
OutputPath string `json:"outputPath,omitempty"`
MetadataPath string `json:"metadataPath,omitempty"`
DataPackagePath string `json:"dataPackagePath,omitempty"`
PreparationPath string `json:"preparationPath,omitempty"`
ExecutionPath string `json:"executionPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"`
GeneratedTextPath string `json:"generatedTextPath,omitempty"`
RenderContextPath string `json:"renderContextPath,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
Command string `json:"command"`
ReportID report.ID `json:"reportId"`
ReportName string `json:"reportName"`
PromptID string `json:"promptId"`
RunID string `json:"runId"`
Status string `json:"status"`
GeneratedAt time.Time `json:"generatedAt"`
ValidPeriod timeutil.Period `json:"validPeriod"`
OutputPath string `json:"outputPath,omitempty"`
LLMDebugPath string `json:"llmDebugPath,omitempty"`
PromptVersion string `json:"promptVersion"`
Timezone string `json:"timezone"`
ProfileID string `json:"profileId,omitempty"`
BackendID string `json:"backendId,omitempty"`
ModelName string `json:"modelName,omitempty"`
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
ValidationStatus string `json:"validationStatus,omitempty"`
Notification *generateNotificationSummary `json:"notification,omitempty"`
Error string `json:"error,omitempty"`
}
type generateNotificationSummary struct {
@@ -73,24 +73,20 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
return summary
}
metadata := result.Metadata
summary.ReportID = metadata.ReportID
summary.ReportName = reportName(metadata.ReportID)
summary.PromptID = metadata.PromptID
summary.RunID = metadata.RunID
summary.ReportID = result.ReportID
summary.ReportName = result.ReportName
summary.PromptID = result.PromptID
summary.PromptVersion = result.PromptVersion
summary.RunID = result.RunID
summary.Status = summaryStatusSucceeded
summary.GeneratedAt = metadata.GeneratedAt
summary.ValidPeriod = metadata.ValidPeriod
summary.ReportPath = result.ReportPath
summary.GeneratedAt = result.GeneratedAt
summary.ValidPeriod = result.ValidPeriod
summary.Timezone = result.Timezone
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
summary.ValidationStatus = string(result.ValidationStatus)
summary.OutputPath = result.OutputPath
summary.MetadataPath = result.MetadataPath
summary.DataPackagePath = result.DataPackagePath
summary.PreparationPath = result.PreparationPath
summary.ExecutionPath = result.ExecutionPath
summary.LLMDebugPath = result.LLMDebugPath
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
summary.GeneratedTextPath = result.GeneratedTextPath
summary.RenderContextPath = result.RenderContextPath
summary.Notification = newGenerateNotificationSummary(result.Notification)
if err != nil {
summary.Status = summaryStatusFailed

251
internal/cli/result_test.go
View File

@@ -2,264 +2,29 @@ package cli
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
acceptedAt := generatedAt.Add(time.Minute)
startedAt := acceptedAt.Add(time.Minute)
finishedAt := startedAt.Add(time.Minute)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ExecutionPath: "/runs/hourly/execution.json",
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json",
GeneratedTextPath: "/runs/hourly/generated_text.json",
RenderContextPath: "/runs/hourly/render_context.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
Notification: &app.NotificationResult{
Status: "succeeded",
UploadStatus: "accepted",
RunID: "distributor-run",
PipelineID: "weatherreporter.hourly",
BundleID: "weatherreporter.home.hourly",
IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly",
AcceptedAt: acceptedAt,
StartedAt: &startedAt,
FinishedAt: &finishedAt,
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
},
}
summary := newGenerateSummary(result, nil)
if summary.Command != "generate" || summary.Status != "succeeded" {
t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status)
}
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
}
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
}
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {
t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification)
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) {
t.Fatalf("summary = %#v", summary)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") {
t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data))
}
if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") {
t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data))
}
}
func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
ReportPath: "/runs/daily/report.md",
OutputPath: "/copies/daily.md",
MetadataPath: "/runs/daily/metadata.json",
Metadata: state.Metadata{
ReportID: report.Daily,
PromptID: "weather.daily_generated_text",
RunID: "20260529T133000Z_daily",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
summary := newGenerateSummary(result, nil)
if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" {
t.Fatalf("summary = %#v, want successful daily summary", summary)
}
if summary.Notification != nil {
t.Fatalf("notification summary = %#v, want omitted", summary.Notification)
}
data, err := json.Marshal(summary)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
for _, omitted := range []string{"notification"} {
if strings.Contains(string(data), omitted) {
t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data))
for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
if strings.Contains(string(data), forbidden) {
t.Fatalf("summary includes %q: %s", forbidden, data)
}
}
}
func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) {
result := &app.ReportResult{
DataPackagePath: "/runs/daily/data_package.yaml",
PreparationPath: "/runs/daily/preparation.json",
Metadata: state.Metadata{
ReportID: report.Daily,
RunID: "20260529T133000Z_daily",
},
}
data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed")))
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
text := string(data)
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} {
if strings.Contains(text, omitted) {
t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text)
}
}
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
t.Fatalf("partial summary omits reached paths:\n%s", text)
}
}
func TestNewGenerateSummaryForNotificationFailure(t *testing.T) {
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
result := &app.ReportResult{
DataPackagePath: "/runs/hourly/data_package.yaml",
PreparationPath: "/runs/hourly/preparation.json",
ReportPath: "/runs/hourly/report.md",
OutputPath: "/copies/hourly.md",
MetadataPath: "/runs/hourly/metadata.json",
Metadata: state.Metadata{
ReportID: report.Hourly,
PromptID: "weather.hourly_generated_text",
RunID: "20260529T133000Z_hourly",
GeneratedAt: generatedAt,
ValidPeriod: testSummaryPeriod(generatedAt),
},
}
err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`)
summary := newGenerateSummary(result, err)
if summary.Status != "failed" || summary.Error != err.Error() {
t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error)
}
if summary.ReportPath == "" || summary.MetadataPath == "" {
t.Fatalf("artifact paths = report %q metadata %q, want retained output provenance", summary.ReportPath, summary.MetadataPath)
}
data, marshalErr := json.Marshal(summary)
if marshalErr != nil {
t.Fatalf("Marshal() error = %v", marshalErr)
}
if strings.Contains(string(data), "notificationPath") {
t.Fatalf("summary JSON includes a notification receipt path:\n%s", data)
}
}
func TestNewBatchSummaryStatusDerivation(t *testing.T) {
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
finishedAt := startedAt.Add(2 * time.Minute)
tests := []struct {
name string
result *app.BatchResult
wantStatus string
wantError string
}{
{
name: "success",
result: &app.BatchResult{
Batch: app.BatchMorning,
StartedAt: startedAt,
FinishedAt: finishedAt,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}},
},
wantStatus: "succeeded",
},
{
name: "report failure",
result: &app.BatchResult{
Batch: app.BatchMorning,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{
{ReportID: report.Today, Status: "succeeded"},
{ReportID: report.Tomorrow, Status: "failed", Error: "render failed"},
},
},
wantStatus: "failed",
wantError: "batch morning failed: 1 of 2 reports failed",
},
{
name: "skipped notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 2,
Succeeded: 1,
Failed: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}},
Notification: &app.BatchNotificationResult{
Status: "skipped",
Reason: "one or more reports failed",
},
},
wantStatus: "failed",
wantError: "batch evening failed: 1 of 2 reports failed",
},
{
name: "failed notification",
result: &app.BatchResult{
Batch: app.BatchEvening,
Total: 1,
Succeeded: 1,
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}},
Notification: &app.BatchNotificationResult{
Status: "failed",
Error: "notify batch evening: upload rejected",
},
},
wantStatus: "failed",
wantError: "batch evening notification failed: notify batch evening: upload rejected",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
summary := newBatchSummary(tt.result)
if summary.Command != "run" || summary.Status != tt.wantStatus {
t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus)
}
if summary.Error != tt.wantError {
t.Fatalf("error = %q, want %q", summary.Error, tt.wantError)
}
if len(summary.Reports) != len(tt.result.Reports) {
t.Fatalf("reports = %#v, want copied report list", summary.Reports)
}
})
}
}
func testSummaryPeriod(start time.Time) timeutil.Period {
return timeutil.Period{
Start: start,
End: start.Add(6 * time.Hour),
}
}

631
internal/cli/root_test.go
View File

@@ -1,631 +0,0 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
)
const (
testRenderedPrompt = "PRIVATE RENDERED PROMPT"
testSchemaBody = `{"private":"schema"}`
testDataBody = "PRIVATE DATA PACKAGE"
testGeneratedBody = "PRIVATE GENERATED BODY"
testEndpoint = "https://user:credential@example.invalid/v1?token=credential"
testParameters = `{"temperature":0.2,"private":"parameter"}`
testCredential = "cli-secret-credential"
)
type commandOutput struct {
stdout string
stderr string
}
type cliExecutor struct {
fail bool
failPrompt string
}
func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text")
return promptexec.PromptInspection{
PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile",
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"},
}, nil
}
func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil
}
func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC)
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath,
StartedAt: now, EndedAt: now,
}
if err := callback(preparation, nil); err != nil {
return nil, err
}
if e.fail || req.PromptID == e.failPrompt {
return nil, errors.New(strings.Join([]string{
"provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters,
}, " "))
}
raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`)
if req.PromptID == "weather.hourly_generated_text" {
raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`)
}
return &promptexec.Execution{
RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash",
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil),
}, nil
}
func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) {
output, err := runCLICommand(Runner{}, "--help")
if err != nil {
t.Fatalf("Run(--help) error = %v", err)
}
for _, command := range []string{
"--version",
"generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening",
"inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources",
} {
if !strings.Contains(output.stdout, command) {
t.Fatalf("help missing %q:\n%s", command, output.stdout)
}
}
for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} {
if strings.Contains(output.stdout, retired) {
t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout)
}
}
for _, description := range []string{"Write the generated Markdown report to PATH.", "Write generated Markdown reports beneath PATH"} {
if !strings.Contains(output.stdout, description) {
t.Fatalf("help missing output description %q:\n%s", description, output.stdout)
}
}
}
func TestRunnerVersion(t *testing.T) {
for _, test := range []struct {
name string
runner Runner
version string
}{
{name: "development default", runner: Runner{}, version: "development"},
{name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"},
} {
t.Run(test.name, func(t *testing.T) {
output, err := runCLICommand(test.runner, "--version")
if err != nil {
t.Fatalf("Run(--version) error = %v", err)
}
if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" {
t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr)
}
})
}
if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil {
t.Fatal("Run(--version extra) error = nil")
}
}
func TestResolveSupportedCommandsAndFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, constructions := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want app.ReportKind
}{
{name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily},
{name: "today", args: []string{"today"}, want: app.ReportToday},
{name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow},
{name: "hourly", args: []string{"hourly"}, want: app.ReportHourly},
} {
t.Run(tt.name, func(t *testing.T) {
args := append(tt.args, "--config", configPath)
req, err := runner.resolveGenerate(args)
if err != nil {
t.Fatalf("resolveGenerate() error = %v", err)
}
if req.Report != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want report %q with executor", req, tt.want)
}
if tt.want == app.ReportDaily || tt.want == app.ReportToday {
if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" {
t.Fatalf("resolved date = %q, want 2026-05-29", got)
}
} else if !req.Date.IsZero() {
t.Fatalf("resolved date = %s, want unset", req.Date)
}
})
}
for _, tt := range []struct {
name string
want app.BatchKind
}{
{name: "morning", want: app.BatchMorning},
{name: "evening", want: app.BatchEvening},
} {
t.Run(tt.name, func(t *testing.T) {
req, err := runner.resolveRun([]string{tt.name, "--config", configPath})
if err != nil {
t.Fatalf("resolveRun() error = %v", err)
}
if req.Batch != tt.want || req.Executor == nil {
t.Fatalf("request = %#v, want batch %q with executor", req, tt.want)
}
})
}
if *constructions != 6 {
t.Fatalf("executor constructions = %d, want one per resolved action", *constructions)
}
}
func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) {
configPath := writeCLIConfig(t, t.TempDir(), "")
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
generate, generateOpts, err := runner.resolveGenerateAction([]string{
"daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC",
"--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveGenerateAction() error = %v", err)
}
if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != filepath.Join(runner.WorkingDir, "daily.md") || generate.WorkingDir != runner.WorkingDir || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet {
t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts)
}
if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" {
t.Fatalf("generate date = %q, want 2026-05-30", got)
}
batch, batchOpts, err := runner.resolveRunAction([]string{
"evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports",
"--llm-debug-dir", "/safe/debug", "--quiet",
})
if err != nil {
t.Fatalf("resolveRunAction() error = %v", err)
}
if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != filepath.Join(runner.WorkingDir, "reports") || batch.WorkingDir != runner.WorkingDir || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet {
t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts)
}
}
func TestGenerateFlagContracts(t *testing.T) {
for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
t.Run(string(kind), func(t *testing.T) {
opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"})
if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" {
t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err)
}
})
}
runner, _ := countingRunner(cliExecutor{})
for _, tt := range []struct {
name string
args []string
want string
}{
{name: "daily requires date", args: []string{"daily"}, want: "requires --date"},
{name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"},
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}},
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}},
{name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}},
{name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}},
{name: "batch rejects output", args: []string{"run", "--out", "report.md"}},
} {
t.Run(tt.name, func(t *testing.T) {
var err error
if tt.args[0] == "run" {
_, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...))
} else {
_, err = runner.resolveGenerate(tt.args)
}
if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) {
t.Fatalf("error = %v, want rejection containing %q", err, tt.want)
}
})
}
}
func TestResolversRejectRetiredAndUnknownNames(t *testing.T) {
runner, _ := countingRunner(cliExecutor{})
for _, name := range []string{"near-term", "three-day", "weekend", "storm"} {
if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") {
t.Fatalf("resolveGenerate(%q) error = %v", name, err)
}
}
for _, name := range []string{"daily", "weekend", "storm"} {
if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") {
t.Fatalf("resolveRun(%q) error = %v", name, err)
}
}
}
func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) {
for _, tt := range []struct {
name string
args func(string, string) []string
}{
{name: "single", args: func(configPath, outputPath string) []string {
return []string{"generate", "today", "--config", configPath, "--out", outputPath}
}},
{name: "batch", args: func(configPath, outputPath string) []string {
return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath}
}},
} {
t.Run(tt.name, func(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
assertRoutineOutputSafe(t, output)
if tt.name == "single" {
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" {
t.Fatalf("single summary = %#v", summary)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout)
}
} else {
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" {
t.Fatalf("batch summary = %#v", summary)
}
if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") {
t.Fatalf("batch status = %q", output.stderr)
}
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout)
}
}
})
}
}
func TestRunnerPreRunFailureAndQuietMode(t *testing.T) {
runner, constructions := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "daily")
if err == nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err)
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions)
}
fixture := newCLIFixture(t)
runner, _ = countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet")
if err != nil || output.stdout != "" || output.stderr != "" {
t.Fatalf("quiet output/error = %#v/%v", output, err)
}
}
func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{fail: true})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want execution failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeGenerateSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" {
t.Fatalf("failed summary paths = %#v", summary)
}
if !strings.Contains(summary.Error, "prompt execution failed") {
t.Fatalf("failed summary error = %q", summary.Error)
}
assertRoutineOutputSafe(t, output)
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID)
if inspectErr != nil {
t.Fatalf("inspect %s error = %v", command, inspectErr)
}
if !strings.Contains(inspected.stdout, summary.RunID) {
t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout)
}
assertRoutineOutputSafe(t, inspected)
}
}
func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) {
fixture := newCLIFixture(t)
runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"})
runner.WorkingDir = t.TempDir()
output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath)
if err == nil {
t.Fatal("Run() error = nil, want aggregate batch failure")
}
if *constructions != 1 {
t.Fatalf("executor constructions = %d, want 1", *constructions)
}
summary := decodeBatchSummary(t, output.stdout)
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 {
t.Fatalf("failed batch summary = %#v", summary)
}
var succeeded, failed *app.BatchReportResult
for index := range summary.Reports {
item := &summary.Reports[index]
if item.Status == summaryStatusSucceeded {
succeeded = item
} else if item.Status == summaryStatusFailed {
failed = item
}
}
if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" {
t.Fatalf("successful batch item paths = %#v", succeeded)
}
if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" {
t.Fatalf("failed batch item paths = %#v", failed)
}
if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") {
t.Fatalf("partial batch status = %q", output.stderr)
}
assertRoutineOutputSafe(t, output)
}
func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runner, _ := countingRunner(cliExecutor{})
runner.WorkingDir = t.TempDir()
first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC))
second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC))
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2")
if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) {
t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err)
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID)
if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) {
t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr)
}
assertRoutineOutputSafe(t, output)
}
prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID)
if err != nil || !strings.Contains(prior.stdout, first.RunID) {
t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err)
}
}
func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) {
fixture := newCLIFixture(t)
runIDs := []string{
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)),
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)),
}
runID := runIDs[0]
runner, _ := countingRunner(cliExecutor{})
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath)
if err != nil {
t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err)
}
for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} {
if !strings.Contains(listed.stdout, want) {
t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout)
}
}
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID)
if inspectErr != nil || !strings.Contains(output.stdout, runID) {
t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr)
}
}
metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID)
if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) {
t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err)
}
}
func countingRunner(executor promptexec.Executor) (Runner, *int) {
count := new(int)
return Runner{
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)},
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
*count++
return executor, nil
},
}, count
}
func runCLICommand(runner Runner, args ...string) (commandOutput, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
err := runner.Run(context.Background(), args, &stdout, &stderr)
return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err
}
func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary {
t.Helper()
base.Clock = timeutil.FixedClock{Time: now}
output, err := runCLICommand(base, "generate", "today", "--config", configPath)
if err != nil {
t.Fatalf("generate current report: %v", err)
}
return decodeGenerateSummary(t, output.stdout)
}
func decodeGenerateSummary(t *testing.T, text string) generateSummary {
t.Helper()
var summary generateSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode generate summary: %v\n%s", err, text)
}
return summary
}
func decodeBatchSummary(t *testing.T, text string) batchSummary {
t.Helper()
var summary batchSummary
if err := json.Unmarshal([]byte(text), &summary); err != nil {
t.Fatalf("decode batch summary: %v\n%s", err, text)
}
return summary
}
func assertRoutineOutputSafe(t *testing.T, output commandOutput) {
t.Helper()
combined := output.stdout + output.stderr
for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} {
if strings.Contains(combined, forbidden) {
t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined)
}
}
}
type cliFixture struct {
tempDir string
workspaceRoot string
configPath string
}
func newCLIFixture(t *testing.T) cliFixture {
t.Helper()
tempDir := t.TempDir()
workspaceRoot := filepath.Join(tempDir, "workspace")
server := weatherServer(t)
return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")}
}
func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) }
func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string {
t.Helper()
configPath := filepath.Join(t.TempDir(), "config.yml")
body := "weather_api:\n timezone: America/Chicago\n"
if baseURL != "" {
body += " base_url: " + baseURL + "\n"
}
body += "workspace:\n root: " + workspaceRoot + "\n"
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
return configPath
}
func weatherServer(t *testing.T) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/observations":
_, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`))
case "/conditions/current":
_, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`))
case "/forecast/hourly":
_, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`))
case "/forecast/narrative":
_, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`))
case "/alerts/active":
_, _ = w.Write([]byte(`{"data":{"alerts":[]}}`))
case "/discussion":
_, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`))
case "/weatherstories/latest":
_, _ = w.Write([]byte(`{"data":null}`))
case "/outlooks/convective":
_, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(server.Close)
return server
}
func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string {
t.Helper()
runID := "historical-" + string(reportID)
date := generatedAt.Format(timeutil.DateLayout)
dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date)
modulePath := filepath.Join(dir, "modules."+runID+".json")
dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml")
metadataPath := filepath.Join(dir, "metadata."+runID+".json")
if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil {
t.Fatalf("create historical fixture directory: %v", err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create historical metadata directory: %v", err)
}
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}})
if err != nil {
t.Fatalf("build historical module snapshot: %v", err)
}
moduleData, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal historical module snapshot: %v", err)
}
if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil {
t.Fatalf("write historical module snapshot: %v", err)
}
period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}
pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{
RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period,
}, Modules: snapshot})
if err != nil {
t.Fatalf("build historical data package: %v", err)
}
data, err := promptinput.MarshalYAML(pkg)
if err != nil {
t.Fatalf("marshal historical data package: %v", err)
}
if err := os.WriteFile(dataPath, data, 0o600); err != nil {
t.Fatalf("write historical data package: %v", err)
}
metadata := state.Metadata{
SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID),
GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive",
ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json",
}
metadataData, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("marshal historical metadata: %v", err)
}
if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil {
t.Fatalf("write historical metadata: %v", err)
}
return runID
}

10
internal/promptdebug/debug_writer.go
View File

@@ -16,8 +16,8 @@ import (
)
const (
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v1"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v1"
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
debugDirectoryMode = 0o700
debugFileMode = 0o600
)
@@ -64,7 +64,6 @@ type PromptDebugPreparation struct {
StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
}
// PromptPreparationDebugArtifact is the on-disk preparation debug record.
@@ -114,7 +113,6 @@ type PromptDebugExecution struct {
StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
}
// PromptExecutionDebugArtifact is the on-disk execution debug record.
@@ -340,7 +338,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
}
}
@@ -351,7 +349,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
}
}

14
internal/promptdebug/debug_writer_test.go
View File

@@ -40,19 +40,19 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
}
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
if !strings.Contains(string(preparationData), want) {
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
}
}
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} {
for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
if !strings.Contains(string(executionData), want) {
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
}
}
for _, data := range [][]byte{preparationData, executionData} {
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") {
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") {
t.Fatalf("debug artifact contains credentials:\n%s", data)
}
}
@@ -169,7 +169,7 @@ func promptDebugPreparationFixture() promptexec.Preparation {
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml",
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
}
}
@@ -180,9 +180,9 @@ func promptDebugExecutionFixture() promptexec.Execution {
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."),
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
RawOutput: []byte("Generated forecast prose."),
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
}
}

16
internal/promptexec/promptexec.go
View File

@@ -61,15 +61,13 @@ type ProfileInspection struct {
}
// ExecuteRequest selects one exact prompt execution. DataPackage is the exact
// YAML input; implementations must copy it before retaining it. DataPackagePath
// is provenance for the inline input, not a provider-readable file reference.
// YAML input; implementations must copy it before retaining it.
type ExecuteRequest struct {
PromptID string
PromptVersion string
ProfileID string
DataPackage []byte
DataPackagePath string
CaptureDebug bool
PromptID string
PromptVersion string
ProfileID string
DataPackage []byte
CaptureDebug bool
}
// PreparationCallback receives safe preparation provenance before provider work.
@@ -90,7 +88,6 @@ type Preparation struct {
StartedAt time.Time
EndedAt time.Time
Duration time.Duration
DataPackagePath string
}
// PreparationDebug contains content-rich preparation details for an explicitly
@@ -127,7 +124,6 @@ type Execution struct {
EndedAt time.Time
Duration time.Duration
Validation Validation
DataPackagePath string
RawOutput []byte
Debug *ExecutionDebug
}

9
internal/promptexec/promptexec_test.go
View File

@@ -42,7 +42,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
if executor.operationalFailure != nil {
return nil, executor.operationalFailure
}
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath}
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion}
var debug *PreparationDebug
if request.CaptureDebug {
debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}}
@@ -57,7 +57,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
if executor.validationRejected {
status = ValidationFailed
}
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
if request.CaptureDebug {
result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")}
}
@@ -65,7 +65,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
}
func TestExecutorLifecycleFixtures(t *testing.T) {
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", DataPackagePath: "data_package.yaml"}
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"}
t.Run("callback failure prevents provider execution", func(t *testing.T) {
executor := &lifecycleExecutor{}
callbackError := errors.New("persistence failed")
@@ -239,7 +239,6 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
BackendID: "local",
ModelName: "model-name",
Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"},
DataPackagePath: "data-packages/daily/data_package.yaml",
}
execution := Execution{
RunID: "run-id",
@@ -254,7 +253,7 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
GeneratedHash: "generated-hash",
RawOutput: []byte("generated content"),
}
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + preparation.DataPackagePath + execution.RunID + execution.GeneratedHash
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash
for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} {
if strings.Contains(text, unwanted) {
t.Fatalf("safe values contain %q: %s", unwanted, text)

24
internal/state/filesystem_test.go
View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
@@ -366,21 +367,40 @@ func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) Promp
artifact.DataPackagePath = paths.DataPackage
artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion
artifact.Preparation.PromptHash = "prompt-hash"
artifact.Preparation.DataPackagePath = paths.DataPackage
return artifact
}
func validPreparationArtifact() PromptPreparationArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptPreparationArtifact{
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
DataPackagePath: "/workspace/data.yaml", Preparation: &promptexec.Preparation{PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0"},
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact {
artifact := validExecutionArtifact()
metadata := resolved.Metadata()
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion
artifact.Provenance.DataPackagePath = paths.DataPackage
artifact.Paths.RawOutputPath = paths.GeneratedTextRaw
return artifact
}
func validExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
Provenance: &PromptExecutionProvenance{RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", BackendID: "backend", ModelName: "model", StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second},
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func writeJSONFixture(t *testing.T, path string, value any) {
t.Helper()
data, err := json.Marshal(value)

5
internal/state/prompt_artifacts.go
View File

@@ -81,7 +81,7 @@ func (a PromptPreparationArtifact) Validate() error {
if a.Preparation == nil || a.Error != nil {
return fmt.Errorf("successful prompt preparation requires preparation without an error")
}
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath {
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion {
return fmt.Errorf("successful prompt preparation provenance must match the artifact")
}
case PromptPreparationFailed:
@@ -122,7 +122,6 @@ type PromptExecutionProvenance struct {
StartedAt time.Time `json:"startedAt"`
EndedAt time.Time `json:"endedAt"`
Duration time.Duration `json:"duration"`
DataPackagePath string `json:"dataPackagePath"`
}
// PromptExecutionPaths records only destinations reached by a completed run.
@@ -165,7 +164,6 @@ func PromptExecutionProvenanceFrom(value promptexec.Execution) PromptExecutionPr
InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID,
ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage,
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
DataPackagePath: value.DataPackagePath,
}
}
@@ -255,7 +253,6 @@ func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error {
}{
{"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash},
{"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName},
{"data package path", value.DataPackagePath},
} {
if strings.TrimSpace(required.value) == "" {
return fmt.Errorf("completed prompt execution provenance %s is required", required.name)

207
internal/state/prompt_artifacts_test.go
View File

@@ -1,207 +0,0 @@
package state
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)
func TestPromptPreparationArtifactValidation(t *testing.T) {
valid := validPreparationArtifact()
if err := valid.Validate(); err != nil {
t.Fatalf("valid successful preparation: %v", err)
}
failed := valid
failed.Status = PromptPreparationFailed
failed.Preparation = nil
failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
if err := failed.Validate(); err != nil {
t.Fatalf("valid failed preparation: %v", err)
}
tests := []struct {
name string
mutate func(*PromptPreparationArtifact)
}{
{"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }},
{"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }},
{"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }},
{"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }},
{"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }},
{"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }},
{"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }},
{"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }},
{"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }},
{"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
{"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }},
{"success error", func(a *PromptPreparationArtifact) {
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
}},
{"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }},
{"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }},
{"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validPreparationArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) {
tests := []struct {
name string
mutate func(*PromptPreparationArtifact)
}{
{"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }},
{"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
{"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }},
{"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }},
{"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validPreparationArtifact()
artifact.Status = PromptPreparationFailed
artifact.Preparation = nil
artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestPromptExecutionArtifactValidation(t *testing.T) {
valid := validExecutionArtifact()
valid.Provenance.RunID = "provider-run-different-from-weatherreporter"
valid.Provenance.GeneratedHash = ""
valid.Provenance.Usage = promptexec.TokenUsage{}
if err := valid.Validate(); err != nil {
t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err)
}
rejected := validExecutionArtifact()
rejected.Status = PromptExecutionValidationRejected
validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"})
rejected.Validation = &validation
if err := rejected.Validate(); err != nil {
t.Fatalf("valid validation rejection: %v", err)
}
failed := validFailedExecutionArtifact()
if err := failed.Validate(); err != nil {
t.Fatalf("valid operational failure: %v", err)
}
tests := []struct {
name string
mutate func(*PromptExecutionArtifact)
}{
{"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }},
{"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }},
{"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }},
{"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }},
{"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }},
{"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }},
{"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }},
{"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
{"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }},
{"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }},
{"wrong validation", func(a *PromptExecutionArtifact) {
value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil)
a.Validation = &value
}},
{"operational error", func(a *PromptExecutionArtifact) {
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
}},
{"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }},
{"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }},
{"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }},
{"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }},
{"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }},
{"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }},
{"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }},
{"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }},
{"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }},
{"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }},
{"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }},
{"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }},
{"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validExecutionArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) {
tests := []struct {
name string
mutate func(*PromptExecutionArtifact)
}{
{"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }},
{"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
{"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }},
{"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact := validFailedExecutionArtifact()
test.mutate(&artifact)
if err := artifact.Validate(); err == nil {
t.Fatalf("Validate() error = nil for %#v", artifact)
}
})
}
}
func validPreparationArtifact() PromptPreparationArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptPreparationArtifact{
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
PromptVersion: "2.0.0", DataPackagePath: "/workspace/data.yaml",
Preparation: &promptexec.Preparation{
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
DataPackagePath: "/workspace/data.yaml",
},
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func validExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
Provenance: &PromptExecutionProvenance{
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
},
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
}
}
func validFailedExecutionArtifact() PromptExecutionArtifact {
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
return PromptExecutionArtifact{
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
}
}