Run report generation without workspace state
This commit is contained in:
@@ -21,7 +21,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
@@ -53,7 +52,6 @@ type GenerateRequest struct {
|
||||
Collector Collector
|
||||
Notifier Notifier
|
||||
Executor promptexec.Executor
|
||||
Store state.Store
|
||||
}
|
||||
|
||||
type BatchRequest struct {
|
||||
@@ -65,7 +63,6 @@ type BatchRequest struct {
|
||||
LLMDebugDir string
|
||||
Collector Collector
|
||||
Executor promptexec.Executor
|
||||
Store state.Store
|
||||
Notifier Notifier
|
||||
}
|
||||
|
||||
@@ -85,21 +82,22 @@ type ReportFacts struct {
|
||||
}
|
||||
|
||||
type ReportResult struct {
|
||||
ModuleSnapshot module.Snapshot
|
||||
ModuleSnapshotPath string
|
||||
DataPackage promptinput.Package
|
||||
DataPackagePath string
|
||||
PreparationPath string
|
||||
ExecutionPath string
|
||||
LLMDebugPath string
|
||||
ReportPath string
|
||||
OutputPath string
|
||||
Metadata state.Metadata
|
||||
MetadataPath string
|
||||
GeneratedTextRawPath string
|
||||
GeneratedTextPath string
|
||||
RenderContextPath string
|
||||
Notification *NotificationResult
|
||||
ReportID report.ID
|
||||
ReportName string
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
RunID string
|
||||
GeneratedAt time.Time
|
||||
Timezone string
|
||||
ValidPeriod timeutil.Period
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
SourceWarnings []weatherdata.SourceWarning
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
LLMDebugPath string
|
||||
OutputPath string
|
||||
Notification *NotificationResult
|
||||
}
|
||||
|
||||
type BatchResult struct {
|
||||
@@ -132,25 +130,26 @@ type BatchNotificationReport struct {
|
||||
}
|
||||
|
||||
type BatchReportResult struct {
|
||||
ReportID report.ID `json:"reportId"`
|
||||
ReportName string `json:"reportName"`
|
||||
PromptID string `json:"promptId"`
|
||||
RunID string `json:"runId"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
NotificationStatus string `json:"notificationStatus,omitempty"`
|
||||
NotificationRunID string `json:"notificationRunId,omitempty"`
|
||||
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
|
||||
NotificationError string `json:"notificationError,omitempty"`
|
||||
GeneratedAt time.Time `json:"generatedAt"`
|
||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
||||
PreparationPath string `json:"preparationPath,omitempty"`
|
||||
ExecutionPath string `json:"executionPath,omitempty"`
|
||||
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||
ReportPath string `json:"reportPath,omitempty"`
|
||||
OutputPath string `json:"outputPath,omitempty"`
|
||||
MetadataPath string `json:"metadataPath,omitempty"`
|
||||
ReportID report.ID `json:"reportId"`
|
||||
ReportName string `json:"reportName"`
|
||||
PromptID string `json:"promptId"`
|
||||
RunID string `json:"runId"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
NotificationStatus string `json:"notificationStatus,omitempty"`
|
||||
NotificationRunID string `json:"notificationRunId,omitempty"`
|
||||
NotificationPipelineID string `json:"notificationPipelineId,omitempty"`
|
||||
NotificationError string `json:"notificationError,omitempty"`
|
||||
GeneratedAt time.Time `json:"generatedAt"`
|
||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||
Timezone string `json:"timezone"`
|
||||
ProfileID string `json:"profileId,omitempty"`
|
||||
BackendID string `json:"backendId,omitempty"`
|
||||
ModelName string `json:"modelName,omitempty"`
|
||||
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
|
||||
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
|
||||
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||
OutputPath string `json:"outputPath,omitempty"`
|
||||
}
|
||||
|
||||
type BatchError struct {
|
||||
@@ -260,14 +259,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := initialReportResult(req, resolved, PromptInspectionResult{})
|
||||
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return result, err
|
||||
}
|
||||
req.OutputPath = outputPath
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||
if err != nil {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
}
|
||||
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
|
||||
Resolved: resolved,
|
||||
@@ -275,11 +275,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
||||
Promptkit: req.Config.Promptkit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return result, err
|
||||
}
|
||||
result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName
|
||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return result, err
|
||||
}
|
||||
return generatePromptReport(ctx, promptReportRequest{
|
||||
GenerateRequest: req,
|
||||
@@ -287,6 +288,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
||||
Collection: *collection,
|
||||
Inspection: inspection,
|
||||
DebugWriter: debugWriter,
|
||||
Result: result,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -339,14 +341,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
return nil, err
|
||||
}
|
||||
if req.Batch == BatchEvening || req.Batch == BatchMorning {
|
||||
store := req.Store
|
||||
if store == nil {
|
||||
defaultStore, err := defaultStore(req.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store = defaultStore
|
||||
}
|
||||
startedAt := now
|
||||
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
|
||||
for _, planned := range plannedReports {
|
||||
@@ -362,7 +356,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
OutputPath: outputPath,
|
||||
Notifier: req.Notifier,
|
||||
Executor: req.Executor,
|
||||
Store: store,
|
||||
},
|
||||
Resolved: resolved,
|
||||
Collection: *collection,
|
||||
@@ -398,13 +391,14 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
}
|
||||
|
||||
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
|
||||
item.DataPackagePath = result.DataPackagePath
|
||||
item.PreparationPath = result.PreparationPath
|
||||
item.ExecutionPath = result.ExecutionPath
|
||||
item.LLMDebugPath = result.LLMDebugPath
|
||||
item.ReportPath = result.ReportPath
|
||||
item.OutputPath = result.OutputPath
|
||||
item.MetadataPath = result.MetadataPath
|
||||
item.ProfileID = result.ProfileID
|
||||
item.BackendID = result.BackendID
|
||||
item.ModelName = result.ModelName
|
||||
item.Timezone = result.Timezone
|
||||
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
|
||||
item.ValidationStatus = result.ValidationStatus
|
||||
if result.Notification != nil {
|
||||
item.NotificationStatus = result.Notification.Status
|
||||
item.NotificationRunID = result.Notification.RunID
|
||||
@@ -451,6 +445,7 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
|
||||
RunID: metadata.RunID,
|
||||
GeneratedAt: metadata.GeneratedAt,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
Timezone: "",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -612,78 +607,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda
|
||||
return bundle, nil
|
||||
}
|
||||
|
||||
type finalizeRenderedReportRequest struct {
|
||||
Config config.Config
|
||||
Store state.Store
|
||||
Resolved report.Resolved
|
||||
Metadata state.Metadata
|
||||
MetadataPath string
|
||||
ExecutionArtifact *state.PromptExecutionArtifact
|
||||
RenderedReportPath string
|
||||
OutputPath string
|
||||
Notifier Notifier
|
||||
GenerationErr error
|
||||
noNotify bool
|
||||
}
|
||||
|
||||
type finalizeRenderedReportResult struct {
|
||||
OutputPath string
|
||||
Metadata state.Metadata
|
||||
MetadataPath string
|
||||
Notification *NotificationResult
|
||||
}
|
||||
|
||||
func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) {
|
||||
if req.Store == nil {
|
||||
return finalizeRenderedReportResult{}, fmt.Errorf("state store is required")
|
||||
}
|
||||
if req.RenderedReportPath == "" {
|
||||
return finalizeRenderedReportResult{}, fmt.Errorf("rendered report path is required for report %q", req.Resolved.Definition.ID)
|
||||
}
|
||||
if req.ExecutionArtifact == nil {
|
||||
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
|
||||
}
|
||||
|
||||
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
|
||||
if req.OutputPath != "" && req.GenerationErr == nil {
|
||||
if req.OutputPath != req.RenderedReportPath {
|
||||
if err := fileutil.CopyFileAtomic(req.RenderedReportPath, req.OutputPath); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.OutputPath = req.OutputPath
|
||||
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
|
||||
paths.OutputPath = req.OutputPath
|
||||
}); err != nil {
|
||||
return result, err
|
||||
}
|
||||
} else {
|
||||
result.OutputPath = req.OutputPath
|
||||
}
|
||||
}
|
||||
|
||||
metadata := req.Metadata
|
||||
metadata.RenderedReportPath = req.RenderedReportPath
|
||||
metadataPath, err := req.Store.SaveMetadata(ctx, metadata)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Metadata = metadata
|
||||
result.MetadataPath = metadataPath
|
||||
if req.GenerationErr != nil {
|
||||
return result, req.GenerationErr
|
||||
}
|
||||
if req.noNotify {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.OutputPath, metadata.RunID, metadata.GeneratedAt, req.Notifier)
|
||||
result.Notification = notification
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
|
||||
notifier, enabled := reportNotifier(cfg, notifier)
|
||||
if !enabled {
|
||||
@@ -965,7 +888,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected
|
||||
}
|
||||
}
|
||||
|
||||
func promptMetadata(metadata state.Metadata) promptinput.Metadata {
|
||||
func promptMetadata(metadata briefing.Metadata) promptinput.Metadata {
|
||||
return promptinput.Metadata{
|
||||
RunID: metadata.RunID,
|
||||
ReportID: metadata.ReportID,
|
||||
@@ -1008,10 +931,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext {
|
||||
return &location
|
||||
}
|
||||
|
||||
func defaultStore(cfg config.Config) (*state.FilesystemStore, error) {
|
||||
return state.NewFilesystemStore(cfg.Workspace)
|
||||
}
|
||||
|
||||
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
batch BatchKind
|
||||
now string
|
||||
wantPrompts int
|
||||
}{
|
||||
{name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3},
|
||||
{name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
cfg.Workspace.Root = t.TempDir()
|
||||
now := mustParse(test.now)
|
||||
req := BatchRequest{Config: cfg, Batch: test.batch, Now: now, WorkingDir: t.TempDir()}
|
||||
candidates, err := batchInspectionCandidates(req, now)
|
||||
if err != nil {
|
||||
t.Fatalf("batchInspectionCandidates() error = %v", err)
|
||||
}
|
||||
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
|
||||
"weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
|
||||
}, prompts: map[string]promptexec.PromptInspection{}}
|
||||
for _, candidate := range candidates {
|
||||
executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition)
|
||||
}
|
||||
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
|
||||
return nil, errors.New("collection reached")
|
||||
})
|
||||
req.Executor = executor
|
||||
req.Collector = collector
|
||||
_, err = RunBatchDetailed(context.Background(), req)
|
||||
if err == nil || err.Error() != "collection reached" {
|
||||
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
|
||||
}
|
||||
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
|
||||
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) {
|
||||
item := BatchReportResult{}
|
||||
copyBatchReportPaths(&item, &ReportResult{
|
||||
DataPackagePath: "/runs/daily/data_package.yaml",
|
||||
PreparationPath: "/runs/daily/preparation.json",
|
||||
})
|
||||
|
||||
data, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} {
|
||||
if strings.Contains(text, omitted) {
|
||||
t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
|
||||
t.Fatalf("batch item omits reached paths:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
|
||||
|
||||
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
|
||||
return f(ctx, req)
|
||||
}
|
||||
|
||||
var _ Collector = collectorFunc(nil)
|
||||
@@ -1,449 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
type assembledBatchExecutor struct {
|
||||
definitions map[string]report.Definition
|
||||
promptRequests []string
|
||||
profileRequests []string
|
||||
executeRequests []promptexec.ExecuteRequest
|
||||
failures map[int]error
|
||||
active int
|
||||
maxActive int
|
||||
}
|
||||
|
||||
func newAssembledBatchExecutor() *assembledBatchExecutor {
|
||||
definitions := make(map[string]report.Definition)
|
||||
for _, definition := range report.DefaultRegistry().All() {
|
||||
definitions[definition.PromptID] = definition
|
||||
}
|
||||
return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)}
|
||||
}
|
||||
|
||||
func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||
e.promptRequests = append(e.promptRequests, id+"@"+version)
|
||||
definition, ok := e.definitions[id]
|
||||
if !ok || definition.PromptVersion != version {
|
||||
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
|
||||
}
|
||||
return logicalPromptInspection(definition), nil
|
||||
}
|
||||
|
||||
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
e.profileRequests = append(e.profileRequests, id)
|
||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||
}
|
||||
|
||||
func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
call := len(e.executeRequests)
|
||||
e.executeRequests = append(e.executeRequests, req)
|
||||
e.active++
|
||||
if e.active > e.maxActive {
|
||||
e.maxActive = e.active
|
||||
}
|
||||
defer func() { e.active-- }()
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
preparation := promptexec.Preparation{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
|
||||
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
|
||||
}
|
||||
if err := callback(preparation, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := e.failures[call]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definition := e.definitions[req.PromptID]
|
||||
return &promptexec.Execution{
|
||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
||||
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
|
||||
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath,
|
||||
RawOutput: []byte(generatedTextForPrompt(req.PromptID)),
|
||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type assembledBatchNotifier struct {
|
||||
reportRequests []NotificationRequest
|
||||
batchRequests []batchNotificationRequest
|
||||
batchResult *NotificationResult
|
||||
batchErr error
|
||||
}
|
||||
|
||||
func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
|
||||
n.reportRequests = append(n.reportRequests, req)
|
||||
return nil, errors.New("per-report notification must be suppressed")
|
||||
}
|
||||
|
||||
func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) {
|
||||
n.batchRequests = append(n.batchRequests, req)
|
||||
if n.batchErr != nil {
|
||||
return nil, n.batchErr
|
||||
}
|
||||
if n.batchResult != nil {
|
||||
result := *n.batchResult
|
||||
if result.PipelineID == "" {
|
||||
result.PipelineID = req.PipelineID
|
||||
}
|
||||
if result.BundleID == "" {
|
||||
result.BundleID = req.BundleID
|
||||
}
|
||||
if result.IdempotencyKey == "" {
|
||||
result.IdempotencyKey = req.IdempotencyKey
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
return &NotificationResult{
|
||||
RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
|
||||
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
batch BatchKind
|
||||
now time.Time
|
||||
wantIDs []report.ID
|
||||
wantCopies []string
|
||||
}{
|
||||
{name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}},
|
||||
{name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, false)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
executor := newAssembledBatchExecutor()
|
||||
outputDir := filepath.Join(t.TempDir(), "output")
|
||||
debugRoot := filepath.Join(t.TempDir(), "debug")
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: test.batch, Now: test.now, WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
|
||||
Collector: collector, Executor: executor,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
||||
}
|
||||
if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 {
|
||||
t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed)
|
||||
}
|
||||
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
|
||||
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
|
||||
}
|
||||
if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
|
||||
t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests)
|
||||
}
|
||||
for index, item := range result.Reports {
|
||||
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {
|
||||
t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index])
|
||||
}
|
||||
if executor.executeRequests[index].PromptID != item.PromptID {
|
||||
t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID)
|
||||
}
|
||||
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" {
|
||||
t.Fatalf("successful report paths = %#v", item)
|
||||
}
|
||||
assertBatchItemMatchesMetadata(t, item)
|
||||
if filepath.Base(item.OutputPath) != test.wantCopies[index] {
|
||||
t.Fatalf("output = %q, want %q", item.OutputPath, test.wantCopies[index])
|
||||
}
|
||||
assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath)
|
||||
renderedReport, readErr := os.ReadFile(item.ReportPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read rendered report: %v", readErr)
|
||||
}
|
||||
copied, readErr := os.ReadFile(item.OutputPath)
|
||||
if readErr != nil || !bytes.Equal(renderedReport, copied) {
|
||||
t.Fatalf("output mismatch/error = %v", readErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, true)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
executor := newAssembledBatchExecutor()
|
||||
executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
|
||||
notifier := &assembledBatchNotifier{}
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(),
|
||||
OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"),
|
||||
Collector: collector, Executor: executor, Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
||||
}
|
||||
if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
||||
t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed)
|
||||
}
|
||||
if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" {
|
||||
t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification)
|
||||
}
|
||||
for index, item := range result.Reports {
|
||||
if index == 1 {
|
||||
if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) {
|
||||
t.Fatalf("failed item = %#v", item)
|
||||
}
|
||||
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" {
|
||||
t.Fatalf("failed item reached paths = %#v", item)
|
||||
}
|
||||
assertBatchItemMatchesMetadata(t, item)
|
||||
continue
|
||||
}
|
||||
if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" {
|
||||
t.Fatalf("continued item %d = %#v", index, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedNotificationLifecycle(t *testing.T) {
|
||||
t.Run("disabled", func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, false)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
notifier := &assembledBatchNotifier{}
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
||||
})
|
||||
if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
|
||||
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("batch disabled", func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, true)
|
||||
cfg.Notify.Distributor.Batch.Enabled = false
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
notifier := &assembledBatchNotifier{}
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
||||
})
|
||||
if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
|
||||
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("all success", func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, true)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
|
||||
RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted",
|
||||
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
|
||||
}}
|
||||
outputDir := filepath.Join(t.TempDir(), "output")
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
||||
}
|
||||
if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 {
|
||||
t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests))
|
||||
}
|
||||
outputPaths := make(map[string]struct{}, len(result.Reports))
|
||||
for _, item := range result.Reports {
|
||||
outputPaths[item.OutputPath] = struct{}{}
|
||||
}
|
||||
request := notifier.batchRequests[0]
|
||||
if len(request.IncludedReports) != len(result.Reports) {
|
||||
t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports))
|
||||
}
|
||||
for _, file := range request.Files {
|
||||
if _, ok := outputPaths[file.SourcePath]; !ok || !strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" {
|
||||
t.Fatalf("notification file = %#v, want selected Markdown output source", file)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("upload failure", func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, true)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")}
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
||||
}
|
||||
if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" {
|
||||
t.Fatalf("result = %#v, want successful reports and failed notification", result)
|
||||
}
|
||||
for _, item := range result.Reports {
|
||||
if item.Status != "succeeded" {
|
||||
t.Fatalf("report item = %#v, want success despite notification failure", item)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(result.Notification.Error, "batch upload rejected") {
|
||||
t.Fatalf("notification error = %#v", result.Notification)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("status report", func(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, true)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
|
||||
RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted",
|
||||
StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
|
||||
}}
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
||||
})
|
||||
if err != nil || result.Failed != 0 || result.Notification == nil {
|
||||
t.Fatalf("result/error = %#v/%v", result, err)
|
||||
}
|
||||
if result.Notification.Status != "accepted" || result.Notification.RunID != "batch-notification-run" {
|
||||
t.Fatalf("notification = %#v", result.Notification)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, false)
|
||||
bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01")
|
||||
outputDir := filepath.Join(t.TempDir(), "output")
|
||||
debugRoot := filepath.Join(t.TempDir(), "debug")
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(),
|
||||
})
|
||||
if err != nil || result.Failed != 0 || len(result.Reports) != 3 {
|
||||
t.Fatalf("result/error = %#v/%v", result, err)
|
||||
}
|
||||
seenRuns := make(map[string]struct{})
|
||||
seenDebug := make(map[string]struct{})
|
||||
dailyDates := make(map[string]BatchReportResult)
|
||||
for _, item := range result.Reports {
|
||||
if _, exists := seenRuns[item.RunID]; exists {
|
||||
t.Fatalf("duplicate run ID %q", item.RunID)
|
||||
}
|
||||
seenRuns[item.RunID] = struct{}{}
|
||||
if _, exists := seenDebug[item.LLMDebugPath]; exists {
|
||||
t.Fatalf("duplicate debug path %q", item.LLMDebugPath)
|
||||
}
|
||||
seenDebug[item.LLMDebugPath] = struct{}{}
|
||||
if item.ReportID == report.Daily {
|
||||
date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02")
|
||||
dailyDates[date] = item
|
||||
}
|
||||
}
|
||||
for _, date := range []string{"2026-05-31", "2026-06-01"} {
|
||||
item, ok := dailyDates[date]
|
||||
if !ok {
|
||||
t.Fatalf("daily items = %#v, want %s", dailyDates, date)
|
||||
}
|
||||
if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") {
|
||||
t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath)
|
||||
}
|
||||
wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID)
|
||||
if item.LLMDebugPath != wantDebugPrefix {
|
||||
t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix)
|
||||
}
|
||||
assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json"))
|
||||
}
|
||||
}
|
||||
|
||||
func assembledBatchConfig(t *testing.T, notify bool) config.Config {
|
||||
t.Helper()
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = notify
|
||||
cfg.Notify.Distributor.Batch.Enabled = notify
|
||||
return cfg
|
||||
}
|
||||
|
||||
func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle {
|
||||
t.Helper()
|
||||
bundle := workflowBundle(t)
|
||||
location := mustLoadTestLocation(t, "America/Chicago")
|
||||
for _, date := range dates {
|
||||
periods := fullDayPeriods(t, date, location)
|
||||
for index := range periods {
|
||||
temperature := float64(60 + index)
|
||||
periods[index].TemperatureF = &temperature
|
||||
periods[index].TextDescription = "Partly cloudy"
|
||||
}
|
||||
bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...)
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
func generatedTextForPrompt(promptID string) string {
|
||||
switch promptID {
|
||||
case "weather.today_generated_text":
|
||||
return validTodayWorkflowJSON()
|
||||
case "weather.tomorrow_generated_text":
|
||||
return validTomorrowWorkflowJSON()
|
||||
case "weather.daily_generated_text":
|
||||
return validDailyWorkflowJSON()
|
||||
case "weather.hourly_generated_text":
|
||||
return validHourlyWorkflowJSON()
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func assertBatchPathsExist(t *testing.T, paths ...string) {
|
||||
t.Helper()
|
||||
for _, path := range paths {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected path %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(item.MetadataPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read metadata %q: %v", item.MetadataPath, err)
|
||||
}
|
||||
var metadata state.Metadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
t.Fatalf("decode metadata %q: %v", item.MetadataPath, err)
|
||||
}
|
||||
if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID ||
|
||||
item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath ||
|
||||
item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath {
|
||||
t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func loadBatchDataPackage(t *testing.T, path string) promptinput.Package {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read data package: %v", err)
|
||||
}
|
||||
pkg, err := promptinput.LoadYAML(data)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadYAML() error = %v", err)
|
||||
}
|
||||
return pkg
|
||||
}
|
||||
103
internal/app/generation_test.go
Normal file
103
internal/app/generation_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
type generationCollector struct {
|
||||
bundle *weatherdata.Bundle
|
||||
err error
|
||||
}
|
||||
|
||||
func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
||||
return &collect.Result{Bundle: c.bundle}, c.err
|
||||
}
|
||||
|
||||
type generationExecutor struct{ called bool }
|
||||
|
||||
func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||
}
|
||||
func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||
}
|
||||
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.called = true
|
||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil
|
||||
}
|
||||
|
||||
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{}
|
||||
workingDir := t.TempDir()
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor})
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||
}
|
||||
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
|
||||
t.Fatalf("workspace state = %v, want absent", err)
|
||||
}
|
||||
data, err := os.ReadFile(result.OutputPath)
|
||||
if err != nil || len(data) == 0 {
|
||||
t.Fatalf("output = %q, error = %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
collectionErr := errors.New("weather source unavailable")
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||
WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{},
|
||||
})
|
||||
if !errors.Is(err, collectionErr) {
|
||||
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
|
||||
}
|
||||
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
|
||||
t.Fatalf("result = %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func generationBundle(t *testing.T) weatherdata.Bundle {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read bundle fixture: %v", err)
|
||||
}
|
||||
var bundle weatherdata.Bundle
|
||||
if err := json.Unmarshal(data, &bundle); err != nil {
|
||||
t.Fatalf("decode bundle fixture: %v", err)
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
func generationTime(value string) time.Time {
|
||||
parsed, _ := time.Parse(time.RFC3339, value)
|
||||
return parsed
|
||||
}
|
||||
|
||||
var _ promptexec.Executor = (*generationExecutor)(nil)
|
||||
var _ Collector = generationCollector{}
|
||||
var _ = report.Daily
|
||||
@@ -33,7 +33,7 @@ type SourceInspection struct {
|
||||
}
|
||||
|
||||
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
|
||||
store, err := defaultStore(req.Config)
|
||||
store, err := state.NewFilesystemStore(req.Config.Workspace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -94,7 +94,7 @@ type runInspection struct {
|
||||
}
|
||||
|
||||
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
|
||||
store, err := defaultStore(req.Config)
|
||||
store, err := state.NewFilesystemStore(req.Config.Workspace)
|
||||
if err != nil {
|
||||
return runInspection{}, err
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
)
|
||||
|
||||
func TestResolveReportOutputPath(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
cfg := config.Defaults()
|
||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
||||
|
||||
for _, test := range []struct {
|
||||
report ReportKind
|
||||
date string
|
||||
name string
|
||||
}{
|
||||
{report: ReportDaily, date: "2026-05-30T12:00:00-05:00", name: "daily-2026-05-30.md"},
|
||||
{report: ReportToday, date: "2026-05-29T12:00:00-05:00", name: "today.md"},
|
||||
{report: ReportTomorrow, name: "tomorrow.md"},
|
||||
{report: ReportHourly, name: "hourly.md"},
|
||||
} {
|
||||
t.Run(string(test.report), func(t *testing.T) {
|
||||
req := GenerateRequest{Config: cfg, Report: test.report}
|
||||
if test.date != "" {
|
||||
req.Date = workflowTime(test.date)
|
||||
}
|
||||
resolved, err := ResolveGenerate(req, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
path, err := resolveReportOutputPath(workingDir, "", resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveReportOutputPath() error = %v", err)
|
||||
}
|
||||
if path != filepath.Join(workingDir, test.name) {
|
||||
t.Fatalf("path = %q, want %q", path, filepath.Join(workingDir, test.name))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
resolved, err := ResolveGenerate(GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-30T12:00:00-05:00"),
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
absoluteDir := t.TempDir()
|
||||
for _, test := range []struct {
|
||||
override string
|
||||
want string
|
||||
}{
|
||||
{override: filepath.Join("reports", "custom.md"), want: filepath.Join(workingDir, "reports", "custom.md")},
|
||||
{override: filepath.Join(absoluteDir, "custom.md"), want: filepath.Join(absoluteDir, "custom.md")},
|
||||
} {
|
||||
path, err := resolveReportOutputPath(workingDir, test.override, resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveReportOutputPath(%q) error = %v", test.override, err)
|
||||
}
|
||||
if path != filepath.Clean(test.want) {
|
||||
t.Fatalf("path = %q, want %q", path, filepath.Clean(test.want))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedRejectsInvalidOutputBeforeCollection(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
collector := &workflowCollector{err: errors.New("collection must not run")}
|
||||
_, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
|
||||
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: t.TempDir(),
|
||||
Collector: collector,
|
||||
})
|
||||
if err == nil || collector.calls != 0 {
|
||||
t.Fatalf("GenerateDetailed() error/calls = %v/%d, want invalid output before collection", err, collector.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedPreservesExistingOutputWhenGenerationFails(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = false
|
||||
workingDir := t.TempDir()
|
||||
outputPath := filepath.Join(workingDir, "daily-2026-05-29.md")
|
||||
if err := os.WriteFile(outputPath, []byte("existing report"), 0o600); err != nil {
|
||||
t.Fatalf("write existing output: %v", err)
|
||||
}
|
||||
bundle := workflowBundle(t)
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
|
||||
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
||||
Executor: &workflowExecutor{definition: report.DefaultRegistry().MustLookup(report.Daily), raw: []byte(`{}`)},
|
||||
})
|
||||
if err == nil || result == nil || result.OutputPath != "" {
|
||||
t.Fatalf("result/error/output = %#v/%v/%q, want failed generation without output publication", result, err, result.OutputPath)
|
||||
}
|
||||
data, readErr := os.ReadFile(outputPath)
|
||||
if readErr != nil || string(data) != "existing report" {
|
||||
t.Fatalf("output after failure = %q, error %v, want preserved content", data, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatchDetailedUsesWorkingDirectoryForOutput(t *testing.T) {
|
||||
cfg := assembledBatchConfig(t, false)
|
||||
workingDir := t.TempDir()
|
||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: workingDir,
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
||||
Executor: newAssembledBatchExecutor(),
|
||||
})
|
||||
if err != nil || result == nil || result.Failed != 0 {
|
||||
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
if len(result.Reports) != 2 {
|
||||
t.Fatalf("reports = %#v, want Tomorrow and Daily", result.Reports)
|
||||
}
|
||||
if result.Reports[0].OutputPath != filepath.Join(workingDir, "tomorrow.md") ||
|
||||
result.Reports[1].OutputPath != filepath.Join(workingDir, "daily-2026-05-31.md") {
|
||||
t.Fatalf("output paths = %#v, want working-directory defaults", result.Reports)
|
||||
}
|
||||
}
|
||||
@@ -1,523 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
const (
|
||||
failPromptExecution = "prompt execution"
|
||||
failMetadata = "metadata"
|
||||
failGeneratedText = "generated text"
|
||||
failRenderContext = "render context"
|
||||
failRenderedReportPath = "rendered report path"
|
||||
)
|
||||
|
||||
type failingPersistenceStore struct {
|
||||
state.Store
|
||||
failOperation string
|
||||
failExecutionCall int
|
||||
failMetadataCall int
|
||||
executionCalls int
|
||||
metadataCalls int
|
||||
renderedReportPath string
|
||||
}
|
||||
|
||||
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
|
||||
s.executionCalls++
|
||||
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
|
||||
return "", errors.New("injected prompt execution persistence failure")
|
||||
}
|
||||
return s.Store.SavePromptExecution(ctx, resolved, artifact)
|
||||
}
|
||||
|
||||
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
|
||||
if s.failOperation == failGeneratedText {
|
||||
return "", errors.New("injected generated text persistence failure")
|
||||
}
|
||||
return s.Store.SaveGeneratedText(ctx, resolved, data)
|
||||
}
|
||||
|
||||
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
|
||||
if s.failOperation == failRenderContext {
|
||||
return "", errors.New("injected render context persistence failure")
|
||||
}
|
||||
return s.Store.SaveRenderContext(ctx, resolved, value)
|
||||
}
|
||||
|
||||
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
|
||||
if s.failOperation == failRenderedReportPath {
|
||||
return s.renderedReportPath, nil
|
||||
}
|
||||
return s.Store.PrepareRenderedReport(ctx, resolved)
|
||||
}
|
||||
|
||||
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
|
||||
s.metadataCalls++
|
||||
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
|
||||
return "", errors.New("injected metadata persistence failure")
|
||||
}
|
||||
return s.Store.SaveMetadata(ctx, metadata)
|
||||
}
|
||||
|
||||
type artifactPathExecutor struct {
|
||||
beforePreparationErr error
|
||||
afterPreparationErr error
|
||||
validation promptexec.ValidationStatus
|
||||
}
|
||||
|
||||
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
||||
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
|
||||
}
|
||||
|
||||
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
||||
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
|
||||
}
|
||||
|
||||
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
if e.beforePreparationErr != nil {
|
||||
return nil, e.beforePreparationErr
|
||||
}
|
||||
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
if err := callback(promptexec.Preparation{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
|
||||
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
|
||||
}, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.afterPreparationErr != nil {
|
||||
return nil, e.afterPreparationErr
|
||||
}
|
||||
validation := e.validation
|
||||
if validation == "" {
|
||||
validation = promptexec.ValidationPassed
|
||||
}
|
||||
return &promptexec.Execution{
|
||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
||||
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
|
||||
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
|
||||
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`),
|
||||
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type successfulNotifier struct{}
|
||||
|
||||
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
||||
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
|
||||
}
|
||||
|
||||
type failingNotifier struct{}
|
||||
|
||||
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
||||
return nil, errors.New("injected notification failure")
|
||||
}
|
||||
|
||||
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
failOperation string
|
||||
failMetadataCall int
|
||||
output bool
|
||||
notify bool
|
||||
want reachedPromptArtifacts
|
||||
}{
|
||||
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
|
||||
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
|
||||
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
|
||||
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
|
||||
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
|
||||
{name: "rendered report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
|
||||
{name: "output then metadata", failOperation: failMetadata, failMetadataCall: 5, output: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
||||
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
|
||||
req.Store = store
|
||||
if test.output {
|
||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
||||
paths.output = req.OutputPath
|
||||
}
|
||||
if test.notify {
|
||||
req.Config.Notify.Distributor.Enabled = true
|
||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
||||
req.Notifier = successfulNotifier{}
|
||||
req.noNotify = false
|
||||
}
|
||||
|
||||
result, err := generatePromptReport(context.Background(), req)
|
||||
if err == nil || result == nil {
|
||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
||||
}
|
||||
assertReachedPromptArtifacts(t, result, paths, test.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
executor artifactPathExecutor
|
||||
want reachedPromptArtifacts
|
||||
wantExecutionStatus state.PromptExecutionStatus
|
||||
wantExecutionPaths state.PromptExecutionPaths
|
||||
wantRawExecution bool
|
||||
}{
|
||||
{
|
||||
name: "preparation failure",
|
||||
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
|
||||
want: reachedPromptArtifacts{preparation: true, metadata: true},
|
||||
},
|
||||
{
|
||||
name: "operational execution failure",
|
||||
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
|
||||
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
|
||||
wantExecutionStatus: state.PromptExecutionFailed,
|
||||
},
|
||||
{
|
||||
name: "completed validation rejection",
|
||||
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
|
||||
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
||||
wantExecutionStatus: state.PromptExecutionValidationRejected,
|
||||
wantRawExecution: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req, paths := promptArtifactRequest(t, test.executor)
|
||||
if test.wantRawExecution {
|
||||
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
|
||||
}
|
||||
result, err := generatePromptReport(context.Background(), req)
|
||||
if err == nil || result == nil {
|
||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
||||
}
|
||||
assertReachedPromptArtifacts(t, result, paths, test.want)
|
||||
if test.wantExecutionStatus != "" {
|
||||
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
|
||||
if loadErr != nil {
|
||||
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
|
||||
}
|
||||
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
|
||||
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
|
||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
||||
paths.output = req.OutputPath
|
||||
req.Config.Notify.Distributor.Enabled = true
|
||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
||||
req.Notifier = successfulNotifier{}
|
||||
req.noNotify = false
|
||||
|
||||
result, err := generatePromptReport(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("generatePromptReport() error = %v", err)
|
||||
}
|
||||
want := state.PromptExecutionPaths{
|
||||
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
|
||||
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
|
||||
OutputPath: paths.output,
|
||||
}
|
||||
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
|
||||
|
||||
data, err := os.ReadFile(result.ExecutionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read execution artifact: %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
for _, forbidden := range []string{
|
||||
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
|
||||
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
|
||||
"credential", "secret-value",
|
||||
} {
|
||||
if strings.Contains(text, forbidden) {
|
||||
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
failOperation string
|
||||
failExecutionCall int
|
||||
failMetadataCall int
|
||||
requestOutput bool
|
||||
failOutput bool
|
||||
notify bool
|
||||
notificationFailure bool
|
||||
wantExecution reachedExecutionArtifacts
|
||||
wantResult reachedPromptArtifacts
|
||||
}{
|
||||
{
|
||||
name: "normalized text write", failOperation: failGeneratedText,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
||||
},
|
||||
{
|
||||
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
||||
},
|
||||
{
|
||||
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
||||
},
|
||||
{
|
||||
name: "render context write", failOperation: failRenderContext,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
||||
},
|
||||
{
|
||||
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
||||
},
|
||||
{
|
||||
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
||||
},
|
||||
{
|
||||
name: "rendered report write", failOperation: failRenderedReportPath,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
||||
},
|
||||
{
|
||||
name: "rendered report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
||||
},
|
||||
{
|
||||
name: "output write", requestOutput: true, failOutput: true,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
||||
},
|
||||
{
|
||||
name: "output checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
||||
},
|
||||
{
|
||||
name: "output metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
||||
},
|
||||
{
|
||||
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
|
||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
||||
store := &failingPersistenceStore{
|
||||
Store: req.Store, failOperation: test.failOperation,
|
||||
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
|
||||
}
|
||||
if test.failOperation == failRenderedReportPath {
|
||||
store.renderedReportPath = t.TempDir()
|
||||
}
|
||||
req.Store = store
|
||||
if test.requestOutput {
|
||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
||||
paths.output = req.OutputPath
|
||||
}
|
||||
if test.failOutput {
|
||||
blocker := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
|
||||
t.Fatalf("write output blocker: %v", err)
|
||||
}
|
||||
req.OutputPath = filepath.Join(blocker, "daily.md")
|
||||
paths.output = req.OutputPath
|
||||
}
|
||||
if test.notify {
|
||||
req.Config.Notify.Distributor.Enabled = true
|
||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
||||
req.Notifier = successfulNotifier{}
|
||||
req.noNotify = false
|
||||
}
|
||||
if test.notificationFailure {
|
||||
req.Notifier = failingNotifier{}
|
||||
}
|
||||
|
||||
result, err := generatePromptReport(context.Background(), req)
|
||||
if err == nil || result == nil {
|
||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
||||
}
|
||||
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
|
||||
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type reachedExecutionArtifacts struct {
|
||||
raw bool
|
||||
normalized bool
|
||||
renderContext bool
|
||||
report bool
|
||||
output bool
|
||||
}
|
||||
|
||||
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
|
||||
result := state.PromptExecutionPaths{}
|
||||
if reached.raw {
|
||||
result.RawOutputPath = paths.GeneratedTextRaw
|
||||
}
|
||||
if reached.normalized {
|
||||
result.GeneratedTextPath = paths.GeneratedText
|
||||
}
|
||||
if reached.renderContext {
|
||||
result.RenderContextPath = paths.RenderContext
|
||||
}
|
||||
if reached.report {
|
||||
result.RenderedReportPath = paths.RenderedReport
|
||||
}
|
||||
if reached.output {
|
||||
result.OutputPath = paths.output
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
|
||||
t.Helper()
|
||||
artifact, err := store.LoadPromptExecution(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPromptExecution() error = %v", err)
|
||||
}
|
||||
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
|
||||
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
|
||||
}
|
||||
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
|
||||
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
|
||||
}
|
||||
if artifact.Paths != want {
|
||||
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
|
||||
}
|
||||
}
|
||||
|
||||
type promptArtifactPaths struct {
|
||||
state.ArtifactPaths
|
||||
output string
|
||||
}
|
||||
|
||||
type reachedPromptArtifacts struct {
|
||||
preparation bool
|
||||
execution bool
|
||||
metadata bool
|
||||
raw bool
|
||||
normalized bool
|
||||
renderContext bool
|
||||
report bool
|
||||
output bool
|
||||
}
|
||||
|
||||
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
|
||||
t.Helper()
|
||||
cfg := config.Defaults()
|
||||
cfg.Workspace.Root = t.TempDir()
|
||||
resolved, err := ResolveGenerate(GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
|
||||
}, mustParse("2026-05-29T05:00:00-05:00"))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read daily fixture: %v", err)
|
||||
}
|
||||
var bundle weatherdata.Bundle
|
||||
if err := json.Unmarshal(bundleData, &bundle); err != nil {
|
||||
t.Fatalf("decode daily fixture: %v", err)
|
||||
}
|
||||
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
paths, err := filesystemStore.Paths(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("Paths() error = %v", err)
|
||||
}
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter("")
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
||||
}
|
||||
return promptReportRequest{
|
||||
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
|
||||
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
|
||||
Inspection: PromptInspectionResult{
|
||||
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
||||
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
|
||||
},
|
||||
DebugWriter: debugWriter, noNotify: true,
|
||||
}, promptArtifactPaths{ArtifactPaths: paths}
|
||||
}
|
||||
|
||||
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
|
||||
t.Helper()
|
||||
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
|
||||
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
|
||||
}
|
||||
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
|
||||
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
|
||||
}
|
||||
checks := []struct {
|
||||
name string
|
||||
got string
|
||||
metadataGot string
|
||||
inMetadata bool
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
|
||||
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
|
||||
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
|
||||
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
|
||||
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
|
||||
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
|
||||
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
|
||||
{"output", result.OutputPath, "", false, paths.output, want.output},
|
||||
}
|
||||
for _, check := range checks {
|
||||
if check.want && check.got != check.path {
|
||||
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
|
||||
}
|
||||
if check.want && check.inMetadata && check.metadataGot != check.path {
|
||||
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
|
||||
}
|
||||
if !check.want && check.got != "" {
|
||||
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
|
||||
}
|
||||
if !check.want && check.inMetadata && check.metadataGot != "" {
|
||||
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
@@ -14,7 +13,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
type promptReportRequest struct {
|
||||
@@ -23,9 +22,23 @@ type promptReportRequest struct {
|
||||
Collection collect.Result
|
||||
Inspection PromptInspectionResult
|
||||
DebugWriter *promptdebug.PromptDebugWriter
|
||||
Result *ReportResult
|
||||
noNotify bool
|
||||
}
|
||||
|
||||
type promptReportWorkflow struct {
|
||||
ctx context.Context
|
||||
req promptReportRequest
|
||||
result *ReportResult
|
||||
briefingMetadata briefing.Metadata
|
||||
reportFacts ReportFacts
|
||||
moduleSnapshot module.Snapshot
|
||||
dataPackage []byte
|
||||
handler generatedtext.Handler
|
||||
debugRef promptdebug.PromptDebugRef
|
||||
callbackFailed bool
|
||||
}
|
||||
|
||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||
workflow, err := newPromptReportWorkflow(ctx, req)
|
||||
if err != nil {
|
||||
@@ -34,369 +47,145 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
|
||||
if err := workflow.buildInputs(); err != nil {
|
||||
return workflow.result, err
|
||||
}
|
||||
|
||||
execution, executeErr := workflow.executePrompt()
|
||||
if executeErr != nil {
|
||||
return workflow.result, workflow.handleExecutionFailure(executeErr)
|
||||
execution, err := workflow.executePrompt()
|
||||
if err != nil {
|
||||
if workflow.callbackFailed {
|
||||
return workflow.result, err
|
||||
}
|
||||
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
|
||||
}
|
||||
if execution == nil {
|
||||
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
|
||||
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
||||
return workflow.result, saveErr
|
||||
}
|
||||
return workflow.result, workflow.reportError("execute prompt", err)
|
||||
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
|
||||
}
|
||||
if err := workflow.persistExecutionDebug(*execution); err != nil {
|
||||
workflow.result.ValidationStatus = execution.Validation.Status
|
||||
if err := workflow.writeExecutionDebug(*execution); err != nil {
|
||||
return workflow.result, err
|
||||
}
|
||||
|
||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
|
||||
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
||||
return workflow.result, saveErr
|
||||
}
|
||||
return workflow.result, workflow.reportError("validate prompt execution", err)
|
||||
}
|
||||
if err := workflow.persistCompletedExecution(*execution); err != nil {
|
||||
return workflow.result, err
|
||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
|
||||
}
|
||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)
|
||||
return workflow.result, workflow.reportError("validate prompt execution", err)
|
||||
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
|
||||
}
|
||||
|
||||
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
|
||||
if err != nil {
|
||||
return workflow.result, err
|
||||
}
|
||||
return workflow.finalizeReport(rendered)
|
||||
}
|
||||
|
||||
type promptReportWorkflow struct {
|
||||
ctx context.Context
|
||||
req promptReportRequest
|
||||
store state.Store
|
||||
result *ReportResult
|
||||
metadata state.Metadata
|
||||
briefingMetadata briefing.Metadata
|
||||
reportFacts ReportFacts
|
||||
moduleSnapshot module.Snapshot
|
||||
dataPackageBytes []byte
|
||||
handler generatedtext.Handler
|
||||
executionArtifact state.PromptExecutionArtifact
|
||||
debugRef promptdebug.PromptDebugRef
|
||||
prepared bool
|
||||
callbackFailed bool
|
||||
return workflow.renderAndPublish(execution.RawOutput)
|
||||
}
|
||||
|
||||
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
||||
if req.Collection.Bundle == nil {
|
||||
return nil, fmt.Errorf("collected weather bundle is required")
|
||||
}
|
||||
store := req.Store
|
||||
var err error
|
||||
if store == nil {
|
||||
store, err = defaultStore(req.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := req.Result
|
||||
if result == nil {
|
||||
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
|
||||
}
|
||||
return &promptReportWorkflow{
|
||||
ctx: ctx, req: req,
|
||||
result: result,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
|
||||
metadata := resolved.Metadata()
|
||||
return &ReportResult{
|
||||
ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name,
|
||||
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
||||
RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName,
|
||||
}
|
||||
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) buildInputs() error {
|
||||
paths, err := w.store.Paths(w.req.Resolved)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result = &ReportResult{}
|
||||
var err error
|
||||
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
||||
if err != nil {
|
||||
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err)
|
||||
return w.reportError("build report facts", err)
|
||||
}
|
||||
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
||||
if err != nil {
|
||||
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err)
|
||||
return w.reportError("build module snapshot", err)
|
||||
}
|
||||
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result.ModuleSnapshot = w.moduleSnapshot
|
||||
w.result.ModuleSnapshotPath = moduleSnapshotPath
|
||||
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
||||
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{
|
||||
ModuleSnapshot: moduleSnapshotPath,
|
||||
Metadata: paths.Metadata,
|
||||
})
|
||||
w.result.Metadata = w.metadata
|
||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
|
||||
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot,
|
||||
})
|
||||
w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
|
||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
|
||||
if err != nil {
|
||||
return w.reportError("build data package", err)
|
||||
}
|
||||
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage)
|
||||
w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
|
||||
if err != nil {
|
||||
return err
|
||||
return w.reportError("marshal data package", err)
|
||||
}
|
||||
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.metadata.DataPackagePath = dataPackagePath
|
||||
w.result.DataPackage = dataPackage
|
||||
w.result.DataPackagePath = dataPackagePath
|
||||
w.result.Metadata = w.metadata
|
||||
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
||||
if err != nil {
|
||||
return w.reportError("lookup generated text catalog", err)
|
||||
}
|
||||
w.debugRef = promptdebug.PromptDebugRef{
|
||||
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
|
||||
}
|
||||
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
||||
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{
|
||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
|
||||
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
|
||||
}, w.persistPreparation)
|
||||
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
|
||||
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||
artifact := state.PromptPreparationArtifact{
|
||||
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded,
|
||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
|
||||
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
|
||||
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
||||
if w.req.DebugWriter == nil {
|
||||
return nil
|
||||
}
|
||||
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
||||
if err != nil {
|
||||
w.callbackFailed = true
|
||||
return err
|
||||
}
|
||||
w.prepared = true
|
||||
w.result.PreparationPath = path
|
||||
w.metadata.PreparationPath = path
|
||||
w.result.Metadata = w.metadata
|
||||
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
||||
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
||||
if err != nil {
|
||||
w.callbackFailed = true
|
||||
return promptDebugWriteError(err)
|
||||
}
|
||||
if debugPath != "" {
|
||||
w.result.LLMDebugPath = debugPath
|
||||
}
|
||||
if err := w.saveMetadata(); err != nil {
|
||||
w.callbackFailed = true
|
||||
return err
|
||||
}
|
||||
w.result.LLMDebugPath = path
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error {
|
||||
if w.callbackFailed {
|
||||
return executeErr
|
||||
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
|
||||
if w.req.DebugWriter == nil {
|
||||
return nil
|
||||
}
|
||||
executeErr = classifiedPromptError("prompt execution failed", executeErr)
|
||||
if !w.prepared {
|
||||
if err := w.persistPreparationFailure(executeErr); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.reportError("prepare prompt", executeErr)
|
||||
}
|
||||
if promptexec.CategoryOf(executeErr) != "" {
|
||||
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.reportError("execute prompt", executeErr)
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
|
||||
startedAt, endedAt := time.Now(), time.Now()
|
||||
artifact := state.PromptPreparationArtifact{
|
||||
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
|
||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
|
||||
Error: state.NewPromptArtifactError(executeErr),
|
||||
}
|
||||
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result.PreparationPath = path
|
||||
w.metadata.PreparationPath = path
|
||||
w.result.Metadata = w.metadata
|
||||
return w.saveMetadata()
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
|
||||
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
|
||||
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result.ExecutionPath = path
|
||||
w.metadata.ExecutionPath = path
|
||||
w.result.Metadata = w.metadata
|
||||
return w.saveMetadata()
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
|
||||
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
||||
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
||||
if err != nil {
|
||||
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
||||
}
|
||||
if debugPath != "" {
|
||||
w.result.LLMDebugPath = debugPath
|
||||
if path != "" {
|
||||
w.result.LLMDebugPath = path
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error {
|
||||
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput)
|
||||
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
|
||||
generatedText, _, err := w.handler.Validate(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
return w.result, w.reportError("validate generated text", err)
|
||||
}
|
||||
w.result.GeneratedTextRawPath = rawPath
|
||||
w.metadata.GeneratedTextRawPath = rawPath
|
||||
w.result.Metadata = w.metadata
|
||||
w.executionArtifact = state.PromptExecutionArtifact{
|
||||
SchemaVersion: state.PromptExecutionSchemaVersion,
|
||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
||||
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
|
||||
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
|
||||
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
|
||||
}
|
||||
if execution.Validation.Status == promptexec.ValidationPassed {
|
||||
w.executionArtifact.Status = state.PromptExecutionSucceeded
|
||||
} else {
|
||||
w.executionArtifact.Status = state.PromptExecutionValidationRejected
|
||||
}
|
||||
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result.ExecutionPath = executionPath
|
||||
w.metadata.ExecutionPath = executionPath
|
||||
w.result.Metadata = w.metadata
|
||||
return w.saveMetadata()
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
|
||||
generatedText, normalized, err := w.handler.Validate(raw)
|
||||
if err != nil {
|
||||
return nil, w.reportError("validate generated text", err)
|
||||
}
|
||||
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.result.GeneratedTextPath = generatedTextPath
|
||||
w.metadata.GeneratedTextPath = generatedTextPath
|
||||
w.result.Metadata = w.metadata
|
||||
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
||||
if err != nil {
|
||||
return nil, w.reportError("build render context", err)
|
||||
return w.result, w.reportError("build render context", err)
|
||||
}
|
||||
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.result.RenderContextPath = renderContextPath
|
||||
w.metadata.RenderContextPath = renderContextPath
|
||||
w.result.Metadata = w.metadata
|
||||
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rendered, err := w.handler.Render(renderContext)
|
||||
if err != nil {
|
||||
return nil, w.reportError("render template", err)
|
||||
return w.result, w.reportError("render template", err)
|
||||
}
|
||||
return rendered, nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) {
|
||||
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved)
|
||||
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
w.result.OutputPath = w.req.OutputPath
|
||||
if w.req.noNotify {
|
||||
return w.result, nil
|
||||
}
|
||||
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
|
||||
w.result.Notification = notification
|
||||
if err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
w.result.ReportPath = reportPath
|
||||
w.metadata.RenderedReportPath = reportPath
|
||||
w.result.Metadata = w.metadata
|
||||
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
|
||||
return w.result, err
|
||||
}
|
||||
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
|
||||
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
|
||||
ExecutionArtifact: &w.executionArtifact, RenderedReportPath: reportPath, OutputPath: w.req.OutputPath,
|
||||
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
|
||||
})
|
||||
w.result.OutputPath = finalized.OutputPath
|
||||
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
|
||||
return w.result, err
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
|
||||
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
|
||||
if err := w.persistReachedPath(update); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.saveMetadata()
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) saveMetadata() error {
|
||||
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.result.Metadata = w.metadata
|
||||
w.result.MetadataPath = path
|
||||
return nil
|
||||
return w.result, nil
|
||||
}
|
||||
|
||||
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
||||
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err)
|
||||
}
|
||||
|
||||
func persistReachedPromptPath(
|
||||
ctx context.Context,
|
||||
store state.Store,
|
||||
resolved report.Resolved,
|
||||
artifact *state.PromptExecutionArtifact,
|
||||
update func(*state.PromptExecutionPaths),
|
||||
) error {
|
||||
update(&artifact.Paths)
|
||||
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
|
||||
return err
|
||||
}
|
||||
|
||||
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
|
||||
now := time.Now()
|
||||
return state.PromptExecutionArtifact{
|
||||
SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed,
|
||||
ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now,
|
||||
Error: state.NewPromptArtifactError(err),
|
||||
}
|
||||
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
|
||||
}
|
||||
|
||||
func classifiedPromptError(operation string, err error) error {
|
||||
@@ -409,5 +198,3 @@ func classifiedPromptError(operation string, err error) error {
|
||||
func promptDebugWriteError(err error) error {
|
||||
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
|
||||
}
|
||||
|
||||
func ptr[T any](value T) *T { return &value }
|
||||
|
||||
@@ -1,671 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
type workflowCollector struct {
|
||||
result *collect.Result
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
||||
c.calls++
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
return c.result, nil
|
||||
}
|
||||
|
||||
type workflowExecutor struct {
|
||||
definition report.Definition
|
||||
raw []byte
|
||||
prompt promptexec.PromptInspection
|
||||
inspectionErr error
|
||||
profile promptexec.ProfileInspection
|
||||
profileErr error
|
||||
beforePreparationErr error
|
||||
afterCallbackErr error
|
||||
afterPreparationErr error
|
||||
validation promptexec.ValidationStatus
|
||||
executeCalls int
|
||||
providerCalls int
|
||||
request promptexec.ExecuteRequest
|
||||
beforeProvider func()
|
||||
preparationDebug *promptexec.PreparationDebug
|
||||
executionDebug *promptexec.ExecutionDebug
|
||||
preparation *promptexec.Preparation
|
||||
execution *promptexec.Execution
|
||||
}
|
||||
|
||||
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||
if e.inspectionErr != nil {
|
||||
return promptexec.PromptInspection{}, e.inspectionErr
|
||||
}
|
||||
if id != e.definition.PromptID || version != e.definition.PromptVersion {
|
||||
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
|
||||
}
|
||||
if e.prompt.PromptID != "" {
|
||||
return e.prompt, nil
|
||||
}
|
||||
return validPromptInspection(e.definition), nil
|
||||
}
|
||||
|
||||
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
if e.profileErr != nil {
|
||||
return promptexec.ProfileInspection{}, e.profileErr
|
||||
}
|
||||
profile := e.profile
|
||||
if profile.ProfileID == "" {
|
||||
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
e.executeCalls++
|
||||
e.request = req
|
||||
if e.beforePreparationErr != nil {
|
||||
return nil, e.beforePreparationErr
|
||||
}
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
profile := e.profile
|
||||
if profile.ProfileID == "" {
|
||||
profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"}
|
||||
}
|
||||
preparation := promptexec.Preparation{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
|
||||
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
|
||||
}
|
||||
e.preparation = &preparation
|
||||
if err := callback(preparation, e.preparationDebug); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.afterCallbackErr != nil {
|
||||
return nil, e.afterCallbackErr
|
||||
}
|
||||
if e.beforeProvider != nil {
|
||||
e.beforeProvider()
|
||||
}
|
||||
e.providerCalls++
|
||||
if e.afterPreparationErr != nil {
|
||||
return nil, e.afterPreparationErr
|
||||
}
|
||||
validation := e.validation
|
||||
if validation == "" {
|
||||
validation = promptexec.ValidationPassed
|
||||
}
|
||||
execution := &promptexec.Execution{
|
||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
||||
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
|
||||
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
|
||||
Debug: e.executionDebug,
|
||||
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
||||
}
|
||||
e.execution = execution
|
||||
return execution, nil
|
||||
}
|
||||
|
||||
type workflowNotifier struct {
|
||||
requests []NotificationRequest
|
||||
err error
|
||||
}
|
||||
|
||||
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
|
||||
n.requests = append(n.requests, req)
|
||||
if n.err != nil {
|
||||
return nil, n.err
|
||||
}
|
||||
return &NotificationResult{
|
||||
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
|
||||
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kind ReportKind
|
||||
id report.ID
|
||||
date time.Time
|
||||
raw string
|
||||
wantOutput string
|
||||
}{
|
||||
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
|
||||
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
|
||||
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
|
||||
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
definition := report.DefaultRegistry().MustLookup(test.id)
|
||||
bundle := workflowBundle(t)
|
||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
||||
notifier := &workflowNotifier{}
|
||||
outputPath := filepath.Join(t.TempDir(), test.name+".md")
|
||||
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||
}
|
||||
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
|
||||
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
|
||||
}
|
||||
if executor.request.PromptVersion != definition.PromptVersion {
|
||||
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
|
||||
}
|
||||
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
|
||||
if storeErr != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
|
||||
}
|
||||
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
|
||||
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
|
||||
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
|
||||
}
|
||||
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
|
||||
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
|
||||
}
|
||||
persisted, readErr := os.ReadFile(result.DataPackagePath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read data package: %v", readErr)
|
||||
}
|
||||
if !bytes.Equal(executor.request.DataPackage, persisted) {
|
||||
t.Fatal("executor data package differs from exact persisted YAML bytes")
|
||||
}
|
||||
renderedReport, readErr := os.ReadFile(result.ReportPath)
|
||||
if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) {
|
||||
t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput)
|
||||
}
|
||||
copied, readErr := os.ReadFile(outputPath)
|
||||
if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath {
|
||||
t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath)
|
||||
}
|
||||
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath {
|
||||
t.Fatalf("notification requests = %#v, want selected output source", notifier.requests)
|
||||
}
|
||||
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
|
||||
if notifier.requests[0].PipelineID != wantPipeline {
|
||||
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
|
||||
}
|
||||
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
|
||||
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
|
||||
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
|
||||
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
|
||||
}
|
||||
renderedName := filepath.Base(result.ReportPath)
|
||||
if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
|
||||
t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kind ReportKind
|
||||
id report.ID
|
||||
raw string
|
||||
override string
|
||||
profile promptexec.ProfileInspection
|
||||
}{
|
||||
{
|
||||
name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(),
|
||||
profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"},
|
||||
},
|
||||
{
|
||||
name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(),
|
||||
profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
|
||||
},
|
||||
{
|
||||
name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile",
|
||||
profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Promptkit.Profile = test.override
|
||||
definition := report.DefaultRegistry().MustLookup(test.id)
|
||||
executor := &workflowExecutor{
|
||||
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
|
||||
}
|
||||
bundle := workflowBundle(t)
|
||||
_, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||
}
|
||||
if executor.request.ProfileID != test.profile.ProfileID {
|
||||
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
|
||||
}
|
||||
if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName {
|
||||
t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
|
||||
}
|
||||
if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName {
|
||||
t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type preparationFailingStore struct {
|
||||
state.Store
|
||||
}
|
||||
|
||||
type renderContextFailingStore struct {
|
||||
state.Store
|
||||
}
|
||||
|
||||
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
|
||||
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for index := range snapshot.Outputs {
|
||||
snapshot.Outputs[index].Value = "invalid module value"
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
|
||||
return "", errors.New("injected preparation persistence failure")
|
||||
}
|
||||
|
||||
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*workflowExecutor)
|
||||
wantCategory promptexec.ErrorCategory
|
||||
wantPreparation bool
|
||||
wantExecution bool
|
||||
wantRaw bool
|
||||
wantProviderCall int
|
||||
}{
|
||||
{name: "preparation", configure: func(e *workflowExecutor) {
|
||||
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
|
||||
}, wantCategory: promptexec.Generation, wantPreparation: true},
|
||||
{name: "credential disappears", configure: func(e *workflowExecutor) {
|
||||
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
|
||||
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
|
||||
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
|
||||
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
|
||||
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "canceled", configure: func(e *workflowExecutor) {
|
||||
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
|
||||
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "deadline", configure: func(e *workflowExecutor) {
|
||||
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
|
||||
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "generation", configure: func(e *workflowExecutor) {
|
||||
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
|
||||
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "operational validation error", configure: func(e *workflowExecutor) {
|
||||
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
|
||||
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
||||
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = false
|
||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
||||
test.configure(executor)
|
||||
bundle := workflowBundle(t)
|
||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: collector, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
|
||||
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
|
||||
}
|
||||
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
|
||||
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
||||
}
|
||||
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
|
||||
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*workflowExecutor)
|
||||
wantCategory promptexec.ErrorCategory
|
||||
}{
|
||||
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
|
||||
{name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration},
|
||||
{name: "malformed profile", configure: func(e *workflowExecutor) {
|
||||
e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret")
|
||||
}, wantCategory: promptexec.InvalidConfiguration},
|
||||
{name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration},
|
||||
{name: "credential", configure: func(e *workflowExecutor) {
|
||||
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
|
||||
}, wantCategory: promptexec.MissingCredential},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||
executor := &workflowExecutor{definition: definition}
|
||||
test.configure(executor)
|
||||
collector := &workflowCollector{err: errors.New("collector must not run")}
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: collector, Executor: executor,
|
||||
})
|
||||
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
|
||||
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
|
||||
}
|
||||
if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") {
|
||||
t.Fatalf("error leaks profile details: %v", err)
|
||||
}
|
||||
entries, readErr := os.ReadDir(cfg.Workspace.Root)
|
||||
if readErr != nil || len(entries) != 0 {
|
||||
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = false
|
||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
||||
bundle := workflowBundle(t)
|
||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
|
||||
})
|
||||
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
|
||||
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = false
|
||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
||||
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
|
||||
resolved, err := ResolveGenerate(request, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
paths, err := filesystem.Paths(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("Paths() error = %v", err)
|
||||
}
|
||||
checked := false
|
||||
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
||||
executor.beforeProvider = func() {
|
||||
checked = true
|
||||
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
|
||||
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
|
||||
}
|
||||
}
|
||||
bundle := workflowBundle(t)
|
||||
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
request.Executor = executor
|
||||
request.Store = filesystem
|
||||
result, err := GenerateDetailed(context.Background(), request)
|
||||
if err != nil || result == nil || !checked || result.OutputPath == "" {
|
||||
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
configure func(*GenerateRequest, *workflowNotifier)
|
||||
wantRaw bool
|
||||
wantNormalized bool
|
||||
wantContext bool
|
||||
wantReport bool
|
||||
wantOutput bool
|
||||
}{
|
||||
{name: "generated text decode", raw: `{`, wantRaw: true},
|
||||
{name: "generated text domain", raw: `{}`, wantRaw: true},
|
||||
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
||||
req.Store = renderContextFailingStore{Store: req.Store}
|
||||
}, wantRaw: true, wantNormalized: true},
|
||||
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
||||
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
|
||||
}, wantRaw: true, wantNormalized: true},
|
||||
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
||||
blocker := filepath.Join(t.TempDir(), "report-blocker")
|
||||
if err := os.Mkdir(blocker, 0o700); err != nil {
|
||||
t.Fatalf("create report blocker: %v", err)
|
||||
}
|
||||
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
|
||||
}, wantRaw: true, wantNormalized: true, wantContext: true},
|
||||
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
|
||||
notifier.err = errors.New("notification rejected")
|
||||
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
||||
notifier := &workflowNotifier{}
|
||||
bundle := workflowBundle(t)
|
||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
req := GenerateRequest{
|
||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
||||
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
||||
Executor: executor, Notifier: notifier, Store: filesystem,
|
||||
}
|
||||
if test.configure != nil {
|
||||
test.configure(&req, notifier)
|
||||
}
|
||||
result, err := GenerateDetailed(context.Background(), req)
|
||||
if err == nil || result == nil {
|
||||
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
|
||||
}
|
||||
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
|
||||
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
|
||||
(result.OutputPath != "") != test.wantOutput {
|
||||
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath)
|
||||
}
|
||||
if test.wantRaw {
|
||||
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
|
||||
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
|
||||
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
|
||||
}
|
||||
}
|
||||
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) {
|
||||
t.Fatalf("notification requests = %#v", notifier.requests)
|
||||
}
|
||||
if test.name == "notification" && result.Metadata.NotificationPath != "" {
|
||||
t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
createCollision func(string, report.Resolved) error
|
||||
wantProviderCalls int
|
||||
wantPreparationFile bool
|
||||
}{
|
||||
{
|
||||
name: "preparation debug",
|
||||
createCollision: func(root string, resolved report.Resolved) error {
|
||||
path := workflowDebugRunPath(root, resolved)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, []byte("not a directory"), 0o600)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
|
||||
createCollision: func(root string, resolved report.Resolved) error {
|
||||
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := workflowConfig(t)
|
||||
cfg.Notify.Distributor.Enabled = false
|
||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
||||
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
|
||||
resolved, err := ResolveGenerate(request, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
||||
}
|
||||
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
|
||||
if err := test.createCollision(debugRoot, resolved); err != nil {
|
||||
t.Fatalf("create debug collision: %v", err)
|
||||
}
|
||||
bundle := workflowBundle(t)
|
||||
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
||||
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
||||
request.Executor = executor
|
||||
request.LLMDebugDir = debugRoot
|
||||
result, err := GenerateDetailed(context.Background(), request)
|
||||
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
|
||||
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
|
||||
}
|
||||
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
|
||||
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
||||
}
|
||||
if test.wantPreparationFile && result.MetadataPath == "" {
|
||||
t.Fatal("execution debug failure lost previously persisted metadata")
|
||||
}
|
||||
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
|
||||
_, statErr := os.Stat(preparationDebug)
|
||||
if (statErr == nil) != test.wantPreparationFile {
|
||||
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func workflowDebugRunPath(root string, resolved report.Resolved) string {
|
||||
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
|
||||
}
|
||||
|
||||
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
|
||||
switch id {
|
||||
case report.Daily:
|
||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
|
||||
case report.Today:
|
||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
|
||||
case report.Tomorrow:
|
||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
|
||||
case report.Hourly:
|
||||
return []string{"hourly/index.md"}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func workflowConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
cfg := config.Defaults()
|
||||
cfg.Workspace.Root = t.TempDir()
|
||||
cfg.WeatherAPI.Timezone = "America/Chicago"
|
||||
cfg.Location.ID = "home"
|
||||
cfg.Location.Name = "Testville"
|
||||
cfg.Location.Region = "MO"
|
||||
cfg.Notify.Distributor.Enabled = true
|
||||
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
|
||||
return cfg
|
||||
}
|
||||
|
||||
func workflowBundle(t *testing.T) weatherdata.Bundle {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read bundle fixture: %v", err)
|
||||
}
|
||||
var bundle weatherdata.Bundle
|
||||
if err := json.Unmarshal(data, &bundle); err != nil {
|
||||
t.Fatalf("decode bundle fixture: %v", err)
|
||||
}
|
||||
future := bundle.Hourly.Periods[0]
|
||||
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
||||
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
|
||||
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
|
||||
futureNarrative := bundle.Narrative.Periods[0]
|
||||
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
||||
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
|
||||
futureNarrative.Name = "Tomorrow"
|
||||
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
|
||||
return bundle
|
||||
}
|
||||
|
||||
func workflowTime(value string) time.Time {
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func validHourlyWorkflowJSON() string {
|
||||
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`
|
||||
}
|
||||
|
||||
func validTomorrowWorkflowJSON() string {
|
||||
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
||||
}
|
||||
|
||||
func validTodayWorkflowJSON() string {
|
||||
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
||||
}
|
||||
|
||||
func validDailyWorkflowJSON() string {
|
||||
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
|
||||
}
|
||||
Reference in New Issue
Block a user