Run report generation without workspace state
This commit is contained in:
@@ -124,9 +124,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
|
|||||||
PromptID: request.PromptID,
|
PromptID: request.PromptID,
|
||||||
PromptVersion: request.PromptVersion,
|
PromptVersion: request.PromptVersion,
|
||||||
ProfileID: request.ProfileID,
|
ProfileID: request.ProfileID,
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{"data_package": promptkit.Inline(string(append([]byte(nil), request.DataPackage...)))},
|
||||||
"data_package": promptkit.InlineWithURI(request.DataPackagePath, string(append([]byte(nil), request.DataPackage...))),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, classifyError(err)
|
return nil, classifyError(err)
|
||||||
@@ -134,7 +132,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
|
|||||||
defer prepared.Discard()
|
defer prepared.Discard()
|
||||||
|
|
||||||
details := prepared.Details()
|
details := prepared.Details()
|
||||||
preparation, debug := preparationValues(details, request.DataPackagePath, request.CaptureDebug)
|
preparation, debug := preparationValues(details, request.CaptureDebug)
|
||||||
if preparedCallback != nil {
|
if preparedCallback != nil {
|
||||||
if err := preparedCallback(preparation, debug); err != nil {
|
if err := preparedCallback(preparation, debug); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -145,7 +143,7 @@ func (adapter *Adapter) Execute(ctx context.Context, request promptexec.ExecuteR
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, classifyError(err)
|
return nil, classifyError(err)
|
||||||
}
|
}
|
||||||
return executionValue(result, request.DataPackagePath, request.CaptureDebug), nil
|
return executionValue(result, request.CaptureDebug), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
|
func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
|
||||||
@@ -156,7 +154,7 @@ func outputContract(value promptkit.OutputContract) promptexec.OutputContract {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preparationValues(value promptkit.PreparedRun, dataPackagePath string, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
|
func preparationValues(value promptkit.PreparedRun, captureDebug bool) (promptexec.Preparation, *promptexec.PreparationDebug) {
|
||||||
preparation := promptexec.Preparation{
|
preparation := promptexec.Preparation{
|
||||||
PromptID: value.PromptID,
|
PromptID: value.PromptID,
|
||||||
PromptVersion: value.PromptVersion,
|
PromptVersion: value.PromptVersion,
|
||||||
@@ -170,7 +168,6 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
|
|||||||
StartedAt: value.StartTime,
|
StartedAt: value.StartTime,
|
||||||
EndedAt: value.EndTime,
|
EndedAt: value.EndTime,
|
||||||
Duration: time.Duration(value.DurationMS) * time.Millisecond,
|
Duration: time.Duration(value.DurationMS) * time.Millisecond,
|
||||||
DataPackagePath: dataPackagePath,
|
|
||||||
}
|
}
|
||||||
if !captureDebug {
|
if !captureDebug {
|
||||||
return preparation, nil
|
return preparation, nil
|
||||||
@@ -186,7 +183,7 @@ func preparationValues(value promptkit.PreparedRun, dataPackagePath string, capt
|
|||||||
return preparation, debug
|
return preparation, debug
|
||||||
}
|
}
|
||||||
|
|
||||||
func executionValue(value *promptkit.RunResult, dataPackagePath string, captureDebug bool) *promptexec.Execution {
|
func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.Execution {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -218,7 +215,6 @@ func executionValue(value *promptkit.RunResult, dataPackagePath string, captureD
|
|||||||
EndedAt: value.EndTime,
|
EndedAt: value.EndTime,
|
||||||
Duration: value.Duration,
|
Duration: value.Duration,
|
||||||
Validation: validation,
|
Validation: validation,
|
||||||
DataPackagePath: dataPackagePath,
|
|
||||||
RawOutput: []byte(value.RawOutput),
|
RawOutput: []byte(value.RawOutput),
|
||||||
}
|
}
|
||||||
if captureDebug {
|
if captureDebug {
|
||||||
|
|||||||
@@ -210,7 +210,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
|
|||||||
callbackCalls := 0
|
callbackCalls := 0
|
||||||
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
result, err := adapter.Execute(context.Background(), request, func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||||
callbackCalls++
|
callbackCalls++
|
||||||
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.DataPackagePath != request.DataPackagePath || preparation.ModelName != "test-model" {
|
if preparation.PromptID != request.PromptID || preparation.PromptVersion != request.PromptVersion || preparation.ModelName != "test-model" {
|
||||||
t.Fatalf("preparation = %#v", preparation)
|
t.Fatalf("preparation = %#v", preparation)
|
||||||
}
|
}
|
||||||
if debug != nil {
|
if debug != nil {
|
||||||
@@ -227,7 +227,7 @@ func TestExecuteUsesPreparedInlineDataPackage(t *testing.T) {
|
|||||||
if callbackCalls != 1 || client.callCount() != 1 {
|
if callbackCalls != 1 || client.callCount() != 1 {
|
||||||
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
|
t.Fatalf("callback/provider calls = %d/%d, want 1/1", callbackCalls, client.callCount())
|
||||||
}
|
}
|
||||||
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content || result.DataPackagePath != request.DataPackagePath {
|
if result == nil || result.Validation.Status != promptexec.ValidationPassed || string(result.RawOutput) != client.response.Content {
|
||||||
t.Fatalf("result = %#v", result)
|
t.Fatalf("result = %#v", result)
|
||||||
}
|
}
|
||||||
if result.Debug != nil {
|
if result.Debug != nil {
|
||||||
@@ -254,7 +254,6 @@ func TestExecuteEmbeddedHourlyProfileThroughPreparedPath(t *testing.T) {
|
|||||||
PromptVersion: "2.0.0",
|
PromptVersion: "2.0.0",
|
||||||
ProfileID: "weather-light",
|
ProfileID: "weather-light",
|
||||||
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
|
DataPackage: []byte("report:\n id: hourly\nbriefing: {}\n"),
|
||||||
DataPackagePath: "data-packages/hourly/data_package.yaml",
|
|
||||||
}
|
}
|
||||||
var preparation promptexec.Preparation
|
var preparation promptexec.Preparation
|
||||||
prepared := false
|
prepared := false
|
||||||
@@ -288,7 +287,7 @@ func TestExecuteUsesExactInlineDataPackageProvenance(t *testing.T) {
|
|||||||
if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
|
if _, err := adapter.Execute(context.Background(), request, nil); err != nil {
|
||||||
t.Fatalf("Execute() error = %v", err)
|
t.Fatalf("Execute() error = %v", err)
|
||||||
}
|
}
|
||||||
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != request.DataPackagePath || reader.ref.Body != string(request.DataPackage) {
|
if reader.ref.Type != promptkit.ArtifactRefInline || reader.ref.URI != "" || reader.ref.Body != string(request.DataPackage) {
|
||||||
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
|
t.Fatalf("artifact ref = %#v, want exact inline data package provenance", reader.ref)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -531,7 +530,6 @@ func testExecuteRequest() promptexec.ExecuteRequest {
|
|||||||
PromptVersion: "2.0.0",
|
PromptVersion: "2.0.0",
|
||||||
ProfileID: "test-profile",
|
ProfileID: "test-profile",
|
||||||
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
|
DataPackage: []byte("report:\n id: daily\nbriefing: {}\n"),
|
||||||
DataPackagePath: "data-packages/daily/data_package.yaml",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
)
|
)
|
||||||
@@ -53,7 +52,6 @@ type GenerateRequest struct {
|
|||||||
Collector Collector
|
Collector Collector
|
||||||
Notifier Notifier
|
Notifier Notifier
|
||||||
Executor promptexec.Executor
|
Executor promptexec.Executor
|
||||||
Store state.Store
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type BatchRequest struct {
|
type BatchRequest struct {
|
||||||
@@ -65,7 +63,6 @@ type BatchRequest struct {
|
|||||||
LLMDebugDir string
|
LLMDebugDir string
|
||||||
Collector Collector
|
Collector Collector
|
||||||
Executor promptexec.Executor
|
Executor promptexec.Executor
|
||||||
Store state.Store
|
|
||||||
Notifier Notifier
|
Notifier Notifier
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,20 +82,21 @@ type ReportFacts struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ReportResult struct {
|
type ReportResult struct {
|
||||||
ModuleSnapshot module.Snapshot
|
ReportID report.ID
|
||||||
ModuleSnapshotPath string
|
ReportName string
|
||||||
DataPackage promptinput.Package
|
PromptID string
|
||||||
DataPackagePath string
|
PromptVersion string
|
||||||
PreparationPath string
|
RunID string
|
||||||
ExecutionPath string
|
GeneratedAt time.Time
|
||||||
|
Timezone string
|
||||||
|
ValidPeriod timeutil.Period
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
ModelName string
|
||||||
|
SourceWarnings []weatherdata.SourceWarning
|
||||||
|
ValidationStatus promptexec.ValidationStatus
|
||||||
LLMDebugPath string
|
LLMDebugPath string
|
||||||
ReportPath string
|
|
||||||
OutputPath string
|
OutputPath string
|
||||||
Metadata state.Metadata
|
|
||||||
MetadataPath string
|
|
||||||
GeneratedTextRawPath string
|
|
||||||
GeneratedTextPath string
|
|
||||||
RenderContextPath string
|
|
||||||
Notification *NotificationResult
|
Notification *NotificationResult
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,13 +142,14 @@ type BatchReportResult struct {
|
|||||||
NotificationError string `json:"notificationError,omitempty"`
|
NotificationError string `json:"notificationError,omitempty"`
|
||||||
GeneratedAt time.Time `json:"generatedAt"`
|
GeneratedAt time.Time `json:"generatedAt"`
|
||||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||||
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
Timezone string `json:"timezone"`
|
||||||
PreparationPath string `json:"preparationPath,omitempty"`
|
ProfileID string `json:"profileId,omitempty"`
|
||||||
ExecutionPath string `json:"executionPath,omitempty"`
|
BackendID string `json:"backendId,omitempty"`
|
||||||
|
ModelName string `json:"modelName,omitempty"`
|
||||||
|
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
|
||||||
|
ValidationStatus promptexec.ValidationStatus `json:"validationStatus,omitempty"`
|
||||||
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||||
ReportPath string `json:"reportPath,omitempty"`
|
|
||||||
OutputPath string `json:"outputPath,omitempty"`
|
OutputPath string `json:"outputPath,omitempty"`
|
||||||
MetadataPath string `json:"metadataPath,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type BatchError struct {
|
type BatchError struct {
|
||||||
@@ -260,14 +259,15 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
result := initialReportResult(req, resolved, PromptInspectionResult{})
|
||||||
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
|
outputPath, err := resolveReportOutputPath(req.WorkingDir, req.OutputPath, resolved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return result, err
|
||||||
}
|
}
|
||||||
req.OutputPath = outputPath
|
req.OutputPath = outputPath
|
||||||
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||||
}
|
}
|
||||||
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
|
inspection, err := InspectPromptExecution(ctx, PromptInspectionRequest{
|
||||||
Resolved: resolved,
|
Resolved: resolved,
|
||||||
@@ -275,11 +275,12 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
|||||||
Promptkit: req.Config.Promptkit,
|
Promptkit: req.Config.Promptkit,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
result.ProfileID, result.BackendID, result.ModelName = inspection.ProfileID, inspection.BackendID, inspection.ModelName
|
||||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return result, err
|
||||||
}
|
}
|
||||||
return generatePromptReport(ctx, promptReportRequest{
|
return generatePromptReport(ctx, promptReportRequest{
|
||||||
GenerateRequest: req,
|
GenerateRequest: req,
|
||||||
@@ -287,6 +288,7 @@ func GenerateDetailed(ctx context.Context, req GenerateRequest) (*ReportResult,
|
|||||||
Collection: *collection,
|
Collection: *collection,
|
||||||
Inspection: inspection,
|
Inspection: inspection,
|
||||||
DebugWriter: debugWriter,
|
DebugWriter: debugWriter,
|
||||||
|
Result: result,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,14 +341,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if req.Batch == BatchEvening || req.Batch == BatchMorning {
|
if req.Batch == BatchEvening || req.Batch == BatchMorning {
|
||||||
store := req.Store
|
|
||||||
if store == nil {
|
|
||||||
defaultStore, err := defaultStore(req.Config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
store = defaultStore
|
|
||||||
}
|
|
||||||
startedAt := now
|
startedAt := now
|
||||||
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
|
result := &BatchResult{Batch: req.Batch, StartedAt: startedAt}
|
||||||
for _, planned := range plannedReports {
|
for _, planned := range plannedReports {
|
||||||
@@ -362,7 +356,6 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
|||||||
OutputPath: outputPath,
|
OutputPath: outputPath,
|
||||||
Notifier: req.Notifier,
|
Notifier: req.Notifier,
|
||||||
Executor: req.Executor,
|
Executor: req.Executor,
|
||||||
Store: store,
|
|
||||||
},
|
},
|
||||||
Resolved: resolved,
|
Resolved: resolved,
|
||||||
Collection: *collection,
|
Collection: *collection,
|
||||||
@@ -398,13 +391,14 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
|||||||
}
|
}
|
||||||
|
|
||||||
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
|
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
|
||||||
item.DataPackagePath = result.DataPackagePath
|
|
||||||
item.PreparationPath = result.PreparationPath
|
|
||||||
item.ExecutionPath = result.ExecutionPath
|
|
||||||
item.LLMDebugPath = result.LLMDebugPath
|
item.LLMDebugPath = result.LLMDebugPath
|
||||||
item.ReportPath = result.ReportPath
|
|
||||||
item.OutputPath = result.OutputPath
|
item.OutputPath = result.OutputPath
|
||||||
item.MetadataPath = result.MetadataPath
|
item.ProfileID = result.ProfileID
|
||||||
|
item.BackendID = result.BackendID
|
||||||
|
item.ModelName = result.ModelName
|
||||||
|
item.Timezone = result.Timezone
|
||||||
|
item.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
|
||||||
|
item.ValidationStatus = result.ValidationStatus
|
||||||
if result.Notification != nil {
|
if result.Notification != nil {
|
||||||
item.NotificationStatus = result.Notification.Status
|
item.NotificationStatus = result.Notification.Status
|
||||||
item.NotificationRunID = result.Notification.RunID
|
item.NotificationRunID = result.Notification.RunID
|
||||||
@@ -451,6 +445,7 @@ func batchReportResult(planned plannedBatchReport) BatchReportResult {
|
|||||||
RunID: metadata.RunID,
|
RunID: metadata.RunID,
|
||||||
GeneratedAt: metadata.GeneratedAt,
|
GeneratedAt: metadata.GeneratedAt,
|
||||||
ValidPeriod: metadata.ValidPeriod,
|
ValidPeriod: metadata.ValidPeriod,
|
||||||
|
Timezone: "",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -612,78 +607,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda
|
|||||||
return bundle, nil
|
return bundle, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type finalizeRenderedReportRequest struct {
|
|
||||||
Config config.Config
|
|
||||||
Store state.Store
|
|
||||||
Resolved report.Resolved
|
|
||||||
Metadata state.Metadata
|
|
||||||
MetadataPath string
|
|
||||||
ExecutionArtifact *state.PromptExecutionArtifact
|
|
||||||
RenderedReportPath string
|
|
||||||
OutputPath string
|
|
||||||
Notifier Notifier
|
|
||||||
GenerationErr error
|
|
||||||
noNotify bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type finalizeRenderedReportResult struct {
|
|
||||||
OutputPath string
|
|
||||||
Metadata state.Metadata
|
|
||||||
MetadataPath string
|
|
||||||
Notification *NotificationResult
|
|
||||||
}
|
|
||||||
|
|
||||||
func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportRequest) (finalizeRenderedReportResult, error) {
|
|
||||||
if req.Store == nil {
|
|
||||||
return finalizeRenderedReportResult{}, fmt.Errorf("state store is required")
|
|
||||||
}
|
|
||||||
if req.RenderedReportPath == "" {
|
|
||||||
return finalizeRenderedReportResult{}, fmt.Errorf("rendered report path is required for report %q", req.Resolved.Definition.ID)
|
|
||||||
}
|
|
||||||
if req.ExecutionArtifact == nil {
|
|
||||||
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
|
|
||||||
if req.OutputPath != "" && req.GenerationErr == nil {
|
|
||||||
if req.OutputPath != req.RenderedReportPath {
|
|
||||||
if err := fileutil.CopyFileAtomic(req.RenderedReportPath, req.OutputPath); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.OutputPath = req.OutputPath
|
|
||||||
if err := persistReachedPromptPath(ctx, req.Store, req.Resolved, req.ExecutionArtifact, func(paths *state.PromptExecutionPaths) {
|
|
||||||
paths.OutputPath = req.OutputPath
|
|
||||||
}); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
result.OutputPath = req.OutputPath
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata := req.Metadata
|
|
||||||
metadata.RenderedReportPath = req.RenderedReportPath
|
|
||||||
metadataPath, err := req.Store.SaveMetadata(ctx, metadata)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.Metadata = metadata
|
|
||||||
result.MetadataPath = metadataPath
|
|
||||||
if req.GenerationErr != nil {
|
|
||||||
return result, req.GenerationErr
|
|
||||||
}
|
|
||||||
if req.noNotify {
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.OutputPath, metadata.RunID, metadata.GeneratedAt, req.Notifier)
|
|
||||||
result.Notification = notification
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
|
func notifyReport(ctx context.Context, cfg config.Config, resolved report.Resolved, outputPath, runID string, generatedAt time.Time, notifier Notifier) (*NotificationResult, error) {
|
||||||
notifier, enabled := reportNotifier(cfg, notifier)
|
notifier, enabled := reportNotifier(cfg, notifier)
|
||||||
if !enabled {
|
if !enabled {
|
||||||
@@ -965,7 +888,7 @@ func briefingBuildContext(cfg config.Config, resolved report.Resolved, collected
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func promptMetadata(metadata state.Metadata) promptinput.Metadata {
|
func promptMetadata(metadata briefing.Metadata) promptinput.Metadata {
|
||||||
return promptinput.Metadata{
|
return promptinput.Metadata{
|
||||||
RunID: metadata.RunID,
|
RunID: metadata.RunID,
|
||||||
ReportID: metadata.ReportID,
|
ReportID: metadata.ReportID,
|
||||||
@@ -1008,10 +931,6 @@ func briefingLocation(cfg config.Config) *briefing.LocationContext {
|
|||||||
return &location
|
return &location
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaultStore(cfg config.Config) (*state.FilesystemStore, error) {
|
|
||||||
return state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
|
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
batch BatchKind
|
|
||||||
now string
|
|
||||||
wantPrompts int
|
|
||||||
}{
|
|
||||||
{name: "morning", batch: BatchMorning, now: "2026-05-29T08:00:00-05:00", wantPrompts: 3},
|
|
||||||
{name: "evening", batch: BatchEvening, now: "2026-05-29T18:00:00-05:00", wantPrompts: 2},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := config.Defaults()
|
|
||||||
cfg.Workspace.Root = t.TempDir()
|
|
||||||
now := mustParse(test.now)
|
|
||||||
req := BatchRequest{Config: cfg, Batch: test.batch, Now: now, WorkingDir: t.TempDir()}
|
|
||||||
candidates, err := batchInspectionCandidates(req, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("batchInspectionCandidates() error = %v", err)
|
|
||||||
}
|
|
||||||
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
|
|
||||||
"weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
|
|
||||||
}, prompts: map[string]promptexec.PromptInspection{}}
|
|
||||||
for _, candidate := range candidates {
|
|
||||||
executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition)
|
|
||||||
}
|
|
||||||
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
|
|
||||||
return nil, errors.New("collection reached")
|
|
||||||
})
|
|
||||||
req.Executor = executor
|
|
||||||
req.Collector = collector
|
|
||||||
_, err = RunBatchDetailed(context.Background(), req)
|
|
||||||
if err == nil || err.Error() != "collection reached" {
|
|
||||||
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
|
|
||||||
}
|
|
||||||
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
|
|
||||||
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyBatchReportPathsLeavesUnreachedPathsEmpty(t *testing.T) {
|
|
||||||
item := BatchReportResult{}
|
|
||||||
copyBatchReportPaths(&item, &ReportResult{
|
|
||||||
DataPackagePath: "/runs/daily/data_package.yaml",
|
|
||||||
PreparationPath: "/runs/daily/preparation.json",
|
|
||||||
})
|
|
||||||
|
|
||||||
data, err := json.Marshal(item)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
text := string(data)
|
|
||||||
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "notificationPath"} {
|
|
||||||
if strings.Contains(text, omitted) {
|
|
||||||
t.Fatalf("batch item includes unreached field %q:\n%s", omitted, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
|
|
||||||
t.Fatalf("batch item omits reached paths:\n%s", text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
|
|
||||||
|
|
||||||
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
|
|
||||||
return f(ctx, req)
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ Collector = collectorFunc(nil)
|
|
||||||
@@ -1,449 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
||||||
)
|
|
||||||
|
|
||||||
type assembledBatchExecutor struct {
|
|
||||||
definitions map[string]report.Definition
|
|
||||||
promptRequests []string
|
|
||||||
profileRequests []string
|
|
||||||
executeRequests []promptexec.ExecuteRequest
|
|
||||||
failures map[int]error
|
|
||||||
active int
|
|
||||||
maxActive int
|
|
||||||
}
|
|
||||||
|
|
||||||
func newAssembledBatchExecutor() *assembledBatchExecutor {
|
|
||||||
definitions := make(map[string]report.Definition)
|
|
||||||
for _, definition := range report.DefaultRegistry().All() {
|
|
||||||
definitions[definition.PromptID] = definition
|
|
||||||
}
|
|
||||||
return &assembledBatchExecutor{definitions: definitions, failures: make(map[int]error)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
|
||||||
e.promptRequests = append(e.promptRequests, id+"@"+version)
|
|
||||||
definition, ok := e.definitions[id]
|
|
||||||
if !ok || definition.PromptVersion != version {
|
|
||||||
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
|
|
||||||
}
|
|
||||||
return logicalPromptInspection(definition), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
||||||
e.profileRequests = append(e.profileRequests, id)
|
|
||||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *assembledBatchExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
||||||
call := len(e.executeRequests)
|
|
||||||
e.executeRequests = append(e.executeRequests, req)
|
|
||||||
e.active++
|
|
||||||
if e.active > e.maxActive {
|
|
||||||
e.maxActive = e.active
|
|
||||||
}
|
|
||||||
defer func() { e.active-- }()
|
|
||||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
preparation := promptexec.Preparation{
|
|
||||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
||||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
|
|
||||||
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
|
|
||||||
}
|
|
||||||
if err := callback(preparation, nil); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := e.failures[call]; err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
definition := e.definitions[req.PromptID]
|
|
||||||
return &promptexec.Execution{
|
|
||||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
||||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
||||||
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
|
|
||||||
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath,
|
|
||||||
RawOutput: []byte(generatedTextForPrompt(req.PromptID)),
|
|
||||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type assembledBatchNotifier struct {
|
|
||||||
reportRequests []NotificationRequest
|
|
||||||
batchRequests []batchNotificationRequest
|
|
||||||
batchResult *NotificationResult
|
|
||||||
batchErr error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *assembledBatchNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
|
|
||||||
n.reportRequests = append(n.reportRequests, req)
|
|
||||||
return nil, errors.New("per-report notification must be suppressed")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *assembledBatchNotifier) NotifyBatch(_ context.Context, req batchNotificationRequest) (*NotificationResult, error) {
|
|
||||||
n.batchRequests = append(n.batchRequests, req)
|
|
||||||
if n.batchErr != nil {
|
|
||||||
return nil, n.batchErr
|
|
||||||
}
|
|
||||||
if n.batchResult != nil {
|
|
||||||
result := *n.batchResult
|
|
||||||
if result.PipelineID == "" {
|
|
||||||
result.PipelineID = req.PipelineID
|
|
||||||
}
|
|
||||||
if result.BundleID == "" {
|
|
||||||
result.BundleID = req.BundleID
|
|
||||||
}
|
|
||||||
if result.IdempotencyKey == "" {
|
|
||||||
result.IdempotencyKey = req.IdempotencyKey
|
|
||||||
}
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
return &NotificationResult{
|
|
||||||
RunID: "batch-notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
|
|
||||||
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
batch BatchKind
|
|
||||||
now time.Time
|
|
||||||
wantIDs []report.ID
|
|
||||||
wantCopies []string
|
|
||||||
}{
|
|
||||||
{name: "morning", batch: BatchMorning, now: workflowTime("2026-05-29T08:00:00-05:00"), wantIDs: []report.ID{report.Today, report.Tomorrow, report.Daily}, wantCopies: []string{"today.md", "tomorrow.md", "daily-2026-05-31.md"}},
|
|
||||||
{name: "evening", batch: BatchEvening, now: workflowTime("2026-05-29T18:00:00-05:00"), wantIDs: []report.ID{report.Tomorrow, report.Daily}, wantCopies: []string{"tomorrow.md", "daily-2026-05-31.md"}},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, false)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
executor := newAssembledBatchExecutor()
|
|
||||||
outputDir := filepath.Join(t.TempDir(), "output")
|
|
||||||
debugRoot := filepath.Join(t.TempDir(), "debug")
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: test.batch, Now: test.now, WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
|
|
||||||
Collector: collector, Executor: executor,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if collector.calls != 1 || result.Total != len(test.wantIDs) || result.Succeeded != len(test.wantIDs) || result.Failed != 0 {
|
|
||||||
t.Fatalf("collection/summary = %d/%d/%d/%d", collector.calls, result.Total, result.Succeeded, result.Failed)
|
|
||||||
}
|
|
||||||
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
|
|
||||||
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
|
|
||||||
}
|
|
||||||
if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
|
|
||||||
t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests)
|
|
||||||
}
|
|
||||||
for index, item := range result.Reports {
|
|
||||||
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {
|
|
||||||
t.Fatalf("report %d = %s/%s, want %s/succeeded", index, item.ReportID, item.Status, test.wantIDs[index])
|
|
||||||
}
|
|
||||||
if executor.executeRequests[index].PromptID != item.PromptID {
|
|
||||||
t.Fatalf("execution %d prompt = %q, want item prompt %q", index, executor.executeRequests[index].PromptID, item.PromptID)
|
|
||||||
}
|
|
||||||
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.ReportPath == "" || item.OutputPath == "" || item.MetadataPath == "" {
|
|
||||||
t.Fatalf("successful report paths = %#v", item)
|
|
||||||
}
|
|
||||||
assertBatchItemMatchesMetadata(t, item)
|
|
||||||
if filepath.Base(item.OutputPath) != test.wantCopies[index] {
|
|
||||||
t.Fatalf("output = %q, want %q", item.OutputPath, test.wantCopies[index])
|
|
||||||
}
|
|
||||||
assertBatchPathsExist(t, item.DataPackagePath, item.PreparationPath, item.ExecutionPath, item.LLMDebugPath, item.ReportPath, item.OutputPath, item.MetadataPath)
|
|
||||||
renderedReport, readErr := os.ReadFile(item.ReportPath)
|
|
||||||
if readErr != nil {
|
|
||||||
t.Fatalf("read rendered report: %v", readErr)
|
|
||||||
}
|
|
||||||
copied, readErr := os.ReadFile(item.OutputPath)
|
|
||||||
if readErr != nil || !bytes.Equal(renderedReport, copied) {
|
|
||||||
t.Fatalf("output mismatch/error = %v", readErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunBatchDetailedContinuesAfterCapacityRejection(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, true)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
executor := newAssembledBatchExecutor()
|
|
||||||
executor.failures[1] = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
|
|
||||||
notifier := &assembledBatchNotifier{}
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
OutputDir: filepath.Join(t.TempDir(), "output"), LLMDebugDir: filepath.Join(t.TempDir(), "debug"),
|
|
||||||
Collector: collector, Executor: executor, Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if collector.calls != 1 || len(executor.executeRequests) != 3 || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
|
||||||
t.Fatalf("collection/execution/summary = %d/%d/%d/%d/%d", collector.calls, len(executor.executeRequests), result.Total, result.Succeeded, result.Failed)
|
|
||||||
}
|
|
||||||
if len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 || result.Notification == nil || result.Notification.Status != "skipped" {
|
|
||||||
t.Fatalf("notification state = reports %d batches %d result %#v", len(notifier.reportRequests), len(notifier.batchRequests), result.Notification)
|
|
||||||
}
|
|
||||||
for index, item := range result.Reports {
|
|
||||||
if index == 1 {
|
|
||||||
if item.ReportID != report.Tomorrow || item.Status != "failed" || !strings.Contains(item.Error, string(promptexec.Capacity)) {
|
|
||||||
t.Fatalf("failed item = %#v", item)
|
|
||||||
}
|
|
||||||
if item.DataPackagePath == "" || item.PreparationPath == "" || item.ExecutionPath == "" || item.LLMDebugPath == "" || item.MetadataPath == "" || item.ReportPath != "" || item.OutputPath != "" {
|
|
||||||
t.Fatalf("failed item reached paths = %#v", item)
|
|
||||||
}
|
|
||||||
assertBatchItemMatchesMetadata(t, item)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if item.Status != "succeeded" || item.ReportPath == "" || item.OutputPath == "" {
|
|
||||||
t.Fatalf("continued item %d = %#v", index, item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunBatchDetailedNotificationLifecycle(t *testing.T) {
|
|
||||||
t.Run("disabled", func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, false)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
notifier := &assembledBatchNotifier{}
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil || result.Notification != nil || result.Failed != 0 || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
|
|
||||||
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("batch disabled", func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, true)
|
|
||||||
cfg.Notify.Distributor.Batch.Enabled = false
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
notifier := &assembledBatchNotifier{}
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil || result.Notification != nil || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 0 {
|
|
||||||
t.Fatalf("result/error/requests = %#v/%v/%d/%d", result, err, len(notifier.reportRequests), len(notifier.batchRequests))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("all success", func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, true)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
|
|
||||||
RunID: "batch-notification-run", Status: "succeeded", UploadStatus: "accepted",
|
|
||||||
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
|
|
||||||
}}
|
|
||||||
outputDir := filepath.Join(t.TempDir(), "output")
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchMorning, Now: workflowTime("2026-05-29T08:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir,
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if result.Failed != 0 || result.Notification == nil || result.Notification.Status != "succeeded" || len(notifier.reportRequests) != 0 || len(notifier.batchRequests) != 1 {
|
|
||||||
t.Fatalf("notification result/requests = %#v/%d/%d", result.Notification, len(notifier.reportRequests), len(notifier.batchRequests))
|
|
||||||
}
|
|
||||||
outputPaths := make(map[string]struct{}, len(result.Reports))
|
|
||||||
for _, item := range result.Reports {
|
|
||||||
outputPaths[item.OutputPath] = struct{}{}
|
|
||||||
}
|
|
||||||
request := notifier.batchRequests[0]
|
|
||||||
if len(request.IncludedReports) != len(result.Reports) {
|
|
||||||
t.Fatalf("included reports = %d, want %d", len(request.IncludedReports), len(result.Reports))
|
|
||||||
}
|
|
||||||
for _, file := range request.Files {
|
|
||||||
if _, ok := outputPaths[file.SourcePath]; !ok || !strings.HasPrefix(file.SourcePath, outputDir+string(filepath.Separator)) || file.BundlePath == "" {
|
|
||||||
t.Fatalf("notification file = %#v, want selected Markdown output source", file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("upload failure", func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, true)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
notifier := &assembledBatchNotifier{batchErr: errors.New("batch upload rejected")}
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("RunBatchDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if result.Succeeded != 2 || result.Failed != 1 || result.Notification == nil || result.Notification.Status != "failed" {
|
|
||||||
t.Fatalf("result = %#v, want successful reports and failed notification", result)
|
|
||||||
}
|
|
||||||
for _, item := range result.Reports {
|
|
||||||
if item.Status != "succeeded" {
|
|
||||||
t.Fatalf("report item = %#v, want success despite notification failure", item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(result.Notification.Error, "batch upload rejected") {
|
|
||||||
t.Fatalf("notification error = %#v", result.Notification)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("status report", func(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, true)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
notifier := &assembledBatchNotifier{batchResult: &NotificationResult{
|
|
||||||
RunID: "batch-notification-run", Status: "accepted", UploadStatus: "accepted",
|
|
||||||
StatusError: "status lookup unavailable", Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
|
|
||||||
}}
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(), Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil || result.Failed != 0 || result.Notification == nil {
|
|
||||||
t.Fatalf("result/error = %#v/%v", result, err)
|
|
||||||
}
|
|
||||||
if result.Notification.Status != "accepted" || result.Notification.RunID != "batch-notification-run" {
|
|
||||||
t.Fatalf("notification = %#v", result.Notification)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunBatchDetailedKeepsDynamicDailyArtifactsDistinct(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, false)
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31", "2026-06-01")
|
|
||||||
outputDir := filepath.Join(t.TempDir(), "output")
|
|
||||||
debugRoot := filepath.Join(t.TempDir(), "debug")
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: t.TempDir(), OutputDir: outputDir, LLMDebugDir: debugRoot,
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: newAssembledBatchExecutor(),
|
|
||||||
})
|
|
||||||
if err != nil || result.Failed != 0 || len(result.Reports) != 3 {
|
|
||||||
t.Fatalf("result/error = %#v/%v", result, err)
|
|
||||||
}
|
|
||||||
seenRuns := make(map[string]struct{})
|
|
||||||
seenDebug := make(map[string]struct{})
|
|
||||||
dailyDates := make(map[string]BatchReportResult)
|
|
||||||
for _, item := range result.Reports {
|
|
||||||
if _, exists := seenRuns[item.RunID]; exists {
|
|
||||||
t.Fatalf("duplicate run ID %q", item.RunID)
|
|
||||||
}
|
|
||||||
seenRuns[item.RunID] = struct{}{}
|
|
||||||
if _, exists := seenDebug[item.LLMDebugPath]; exists {
|
|
||||||
t.Fatalf("duplicate debug path %q", item.LLMDebugPath)
|
|
||||||
}
|
|
||||||
seenDebug[item.LLMDebugPath] = struct{}{}
|
|
||||||
if item.ReportID == report.Daily {
|
|
||||||
date := item.ValidPeriod.Start.In(mustLoadTestLocation(t, "America/Chicago")).Format("2006-01-02")
|
|
||||||
dailyDates[date] = item
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, date := range []string{"2026-05-31", "2026-06-01"} {
|
|
||||||
item, ok := dailyDates[date]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("daily items = %#v, want %s", dailyDates, date)
|
|
||||||
}
|
|
||||||
if !strings.HasSuffix(item.RunID, "_daily_"+date) || item.OutputPath != filepath.Join(outputDir, "daily-"+date+".md") {
|
|
||||||
t.Fatalf("daily identity/output = %q/%q", item.RunID, item.OutputPath)
|
|
||||||
}
|
|
||||||
wantDebugPrefix := filepath.Join(debugRoot, "daily", date, item.RunID)
|
|
||||||
if item.LLMDebugPath != wantDebugPrefix {
|
|
||||||
t.Fatalf("daily debug path = %q, want %q", item.LLMDebugPath, wantDebugPrefix)
|
|
||||||
}
|
|
||||||
assertBatchPathsExist(t, filepath.Join(item.LLMDebugPath, "preparation.json"), filepath.Join(item.LLMDebugPath, "execution.json"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assembledBatchConfig(t *testing.T, notify bool) config.Config {
|
|
||||||
t.Helper()
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = notify
|
|
||||||
cfg.Notify.Distributor.Batch.Enabled = notify
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func assembledBatchBundle(t *testing.T, dates ...string) weatherdata.Bundle {
|
|
||||||
t.Helper()
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
location := mustLoadTestLocation(t, "America/Chicago")
|
|
||||||
for _, date := range dates {
|
|
||||||
periods := fullDayPeriods(t, date, location)
|
|
||||||
for index := range periods {
|
|
||||||
temperature := float64(60 + index)
|
|
||||||
periods[index].TemperatureF = &temperature
|
|
||||||
periods[index].TextDescription = "Partly cloudy"
|
|
||||||
}
|
|
||||||
bundle.Hourly.Periods = append(bundle.Hourly.Periods, periods...)
|
|
||||||
}
|
|
||||||
return bundle
|
|
||||||
}
|
|
||||||
|
|
||||||
func generatedTextForPrompt(promptID string) string {
|
|
||||||
switch promptID {
|
|
||||||
case "weather.today_generated_text":
|
|
||||||
return validTodayWorkflowJSON()
|
|
||||||
case "weather.tomorrow_generated_text":
|
|
||||||
return validTomorrowWorkflowJSON()
|
|
||||||
case "weather.daily_generated_text":
|
|
||||||
return validDailyWorkflowJSON()
|
|
||||||
case "weather.hourly_generated_text":
|
|
||||||
return validHourlyWorkflowJSON()
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertBatchPathsExist(t *testing.T, paths ...string) {
|
|
||||||
t.Helper()
|
|
||||||
for _, path := range paths {
|
|
||||||
if _, err := os.Stat(path); err != nil {
|
|
||||||
t.Fatalf("expected path %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertBatchItemMatchesMetadata(t *testing.T, item BatchReportResult) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(item.MetadataPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read metadata %q: %v", item.MetadataPath, err)
|
|
||||||
}
|
|
||||||
var metadata state.Metadata
|
|
||||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
|
||||||
t.Fatalf("decode metadata %q: %v", item.MetadataPath, err)
|
|
||||||
}
|
|
||||||
if item.ReportID != metadata.ReportID || item.RunID != metadata.RunID ||
|
|
||||||
item.DataPackagePath != metadata.DataPackagePath || item.PreparationPath != metadata.PreparationPath ||
|
|
||||||
item.ExecutionPath != metadata.ExecutionPath || item.ReportPath != metadata.RenderedReportPath {
|
|
||||||
t.Fatalf("batch item paths do not exactly match metadata: item=%#v metadata=%#v", item, metadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadBatchDataPackage(t *testing.T, path string) promptinput.Package {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read data package: %v", err)
|
|
||||||
}
|
|
||||||
pkg, err := promptinput.LoadYAML(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadYAML() error = %v", err)
|
|
||||||
}
|
|
||||||
return pkg
|
|
||||||
}
|
|
||||||
103
internal/app/generation_test.go
Normal file
103
internal/app/generation_test.go
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
|
)
|
||||||
|
|
||||||
|
type generationCollector struct {
|
||||||
|
bundle *weatherdata.Bundle
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
||||||
|
return &collect.Result{Bundle: c.bundle}, c.err
|
||||||
|
}
|
||||||
|
|
||||||
|
type generationExecutor struct{ called bool }
|
||||||
|
|
||||||
|
func (generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||||
|
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
||||||
|
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||||
|
}
|
||||||
|
func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||||
|
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||||
|
}
|
||||||
|
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||||
|
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
|
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e.called = true
|
||||||
|
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`), Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
||||||
|
cfg := config.Defaults()
|
||||||
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||||
|
bundle := generationBundle(t)
|
||||||
|
executor := &generationExecutor{}
|
||||||
|
workingDir := t.TempDir()
|
||||||
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: generationCollector{bundle: &bundle}, Executor: executor})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateDetailed() error = %v", err)
|
||||||
|
}
|
||||||
|
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
||||||
|
t.Fatalf("result = %#v", result)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("workspace state = %v, want absent", err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(result.OutputPath)
|
||||||
|
if err != nil || len(data) == 0 {
|
||||||
|
t.Fatalf("output = %q, error = %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
|
||||||
|
cfg := config.Defaults()
|
||||||
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||||
|
collectionErr := errors.New("weather source unavailable")
|
||||||
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
||||||
|
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
||||||
|
WorkingDir: t.TempDir(), Collector: generationCollector{err: collectionErr}, Executor: &generationExecutor{},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, collectionErr) {
|
||||||
|
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
|
||||||
|
}
|
||||||
|
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
|
||||||
|
t.Fatalf("result = %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func generationBundle(t *testing.T) weatherdata.Bundle {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read bundle fixture: %v", err)
|
||||||
|
}
|
||||||
|
var bundle weatherdata.Bundle
|
||||||
|
if err := json.Unmarshal(data, &bundle); err != nil {
|
||||||
|
t.Fatalf("decode bundle fixture: %v", err)
|
||||||
|
}
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
|
func generationTime(value string) time.Time {
|
||||||
|
parsed, _ := time.Parse(time.RFC3339, value)
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ promptexec.Executor = (*generationExecutor)(nil)
|
||||||
|
var _ Collector = generationCollector{}
|
||||||
|
var _ = report.Daily
|
||||||
@@ -33,7 +33,7 @@ type SourceInspection struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
|
func InspectReports(ctx context.Context, req InspectReportsRequest) ([]state.ReportRecord, error) {
|
||||||
store, err := defaultStore(req.Config)
|
store, err := state.NewFilesystemStore(req.Config.Workspace)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -94,7 +94,7 @@ type runInspection struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
|
func inspectRun(ctx context.Context, req InspectRunRequest) (runInspection, error) {
|
||||||
store, err := defaultStore(req.Config)
|
store, err := state.NewFilesystemStore(req.Config.Workspace)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return runInspection{}, err
|
return runInspection{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,129 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveReportOutputPath(t *testing.T) {
|
|
||||||
workingDir := t.TempDir()
|
|
||||||
cfg := config.Defaults()
|
|
||||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
|
||||||
|
|
||||||
for _, test := range []struct {
|
|
||||||
report ReportKind
|
|
||||||
date string
|
|
||||||
name string
|
|
||||||
}{
|
|
||||||
{report: ReportDaily, date: "2026-05-30T12:00:00-05:00", name: "daily-2026-05-30.md"},
|
|
||||||
{report: ReportToday, date: "2026-05-29T12:00:00-05:00", name: "today.md"},
|
|
||||||
{report: ReportTomorrow, name: "tomorrow.md"},
|
|
||||||
{report: ReportHourly, name: "hourly.md"},
|
|
||||||
} {
|
|
||||||
t.Run(string(test.report), func(t *testing.T) {
|
|
||||||
req := GenerateRequest{Config: cfg, Report: test.report}
|
|
||||||
if test.date != "" {
|
|
||||||
req.Date = workflowTime(test.date)
|
|
||||||
}
|
|
||||||
resolved, err := ResolveGenerate(req, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
path, err := resolveReportOutputPath(workingDir, "", resolved)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveReportOutputPath() error = %v", err)
|
|
||||||
}
|
|
||||||
if path != filepath.Join(workingDir, test.name) {
|
|
||||||
t.Fatalf("path = %q, want %q", path, filepath.Join(workingDir, test.name))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-30T12:00:00-05:00"),
|
|
||||||
}, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
absoluteDir := t.TempDir()
|
|
||||||
for _, test := range []struct {
|
|
||||||
override string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{override: filepath.Join("reports", "custom.md"), want: filepath.Join(workingDir, "reports", "custom.md")},
|
|
||||||
{override: filepath.Join(absoluteDir, "custom.md"), want: filepath.Join(absoluteDir, "custom.md")},
|
|
||||||
} {
|
|
||||||
path, err := resolveReportOutputPath(workingDir, test.override, resolved)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveReportOutputPath(%q) error = %v", test.override, err)
|
|
||||||
}
|
|
||||||
if path != filepath.Clean(test.want) {
|
|
||||||
t.Fatalf("path = %q, want %q", path, filepath.Clean(test.want))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedRejectsInvalidOutputBeforeCollection(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
collector := &workflowCollector{err: errors.New("collection must not run")}
|
|
||||||
_, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
|
|
||||||
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: t.TempDir(),
|
|
||||||
Collector: collector,
|
|
||||||
})
|
|
||||||
if err == nil || collector.calls != 0 {
|
|
||||||
t.Fatalf("GenerateDetailed() error/calls = %v/%d, want invalid output before collection", err, collector.calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedPreservesExistingOutputWhenGenerationFails(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = false
|
|
||||||
workingDir := t.TempDir()
|
|
||||||
outputPath := filepath.Join(workingDir, "daily-2026-05-29.md")
|
|
||||||
if err := os.WriteFile(outputPath, []byte("existing report"), 0o600); err != nil {
|
|
||||||
t.Fatalf("write existing output: %v", err)
|
|
||||||
}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"),
|
|
||||||
Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir,
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
|
||||||
Executor: &workflowExecutor{definition: report.DefaultRegistry().MustLookup(report.Daily), raw: []byte(`{}`)},
|
|
||||||
})
|
|
||||||
if err == nil || result == nil || result.OutputPath != "" {
|
|
||||||
t.Fatalf("result/error/output = %#v/%v/%q, want failed generation without output publication", result, err, result.OutputPath)
|
|
||||||
}
|
|
||||||
data, readErr := os.ReadFile(outputPath)
|
|
||||||
if readErr != nil || string(data) != "existing report" {
|
|
||||||
t.Fatalf("output after failure = %q, error %v, want preserved content", data, readErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunBatchDetailedUsesWorkingDirectoryForOutput(t *testing.T) {
|
|
||||||
cfg := assembledBatchConfig(t, false)
|
|
||||||
workingDir := t.TempDir()
|
|
||||||
bundle := assembledBatchBundle(t, "2026-05-31")
|
|
||||||
result, err := RunBatchDetailed(context.Background(), BatchRequest{
|
|
||||||
Config: cfg, Batch: BatchEvening, Now: workflowTime("2026-05-29T18:00:00-05:00"), WorkingDir: workingDir,
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
|
||||||
Executor: newAssembledBatchExecutor(),
|
|
||||||
})
|
|
||||||
if err != nil || result == nil || result.Failed != 0 {
|
|
||||||
t.Fatalf("RunBatchDetailed() result/error = %#v/%v", result, err)
|
|
||||||
}
|
|
||||||
if len(result.Reports) != 2 {
|
|
||||||
t.Fatalf("reports = %#v, want Tomorrow and Daily", result.Reports)
|
|
||||||
}
|
|
||||||
if result.Reports[0].OutputPath != filepath.Join(workingDir, "tomorrow.md") ||
|
|
||||||
result.Reports[1].OutputPath != filepath.Join(workingDir, "daily-2026-05-31.md") {
|
|
||||||
t.Fatalf("output paths = %#v, want working-directory defaults", result.Reports)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,523 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
failPromptExecution = "prompt execution"
|
|
||||||
failMetadata = "metadata"
|
|
||||||
failGeneratedText = "generated text"
|
|
||||||
failRenderContext = "render context"
|
|
||||||
failRenderedReportPath = "rendered report path"
|
|
||||||
)
|
|
||||||
|
|
||||||
type failingPersistenceStore struct {
|
|
||||||
state.Store
|
|
||||||
failOperation string
|
|
||||||
failExecutionCall int
|
|
||||||
failMetadataCall int
|
|
||||||
executionCalls int
|
|
||||||
metadataCalls int
|
|
||||||
renderedReportPath string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
|
|
||||||
s.executionCalls++
|
|
||||||
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
|
|
||||||
return "", errors.New("injected prompt execution persistence failure")
|
|
||||||
}
|
|
||||||
return s.Store.SavePromptExecution(ctx, resolved, artifact)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
|
|
||||||
if s.failOperation == failGeneratedText {
|
|
||||||
return "", errors.New("injected generated text persistence failure")
|
|
||||||
}
|
|
||||||
return s.Store.SaveGeneratedText(ctx, resolved, data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
|
|
||||||
if s.failOperation == failRenderContext {
|
|
||||||
return "", errors.New("injected render context persistence failure")
|
|
||||||
}
|
|
||||||
return s.Store.SaveRenderContext(ctx, resolved, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
|
|
||||||
if s.failOperation == failRenderedReportPath {
|
|
||||||
return s.renderedReportPath, nil
|
|
||||||
}
|
|
||||||
return s.Store.PrepareRenderedReport(ctx, resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
|
|
||||||
s.metadataCalls++
|
|
||||||
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
|
|
||||||
return "", errors.New("injected metadata persistence failure")
|
|
||||||
}
|
|
||||||
return s.Store.SaveMetadata(ctx, metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
type artifactPathExecutor struct {
|
|
||||||
beforePreparationErr error
|
|
||||||
afterPreparationErr error
|
|
||||||
validation promptexec.ValidationStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
|
||||||
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
|
||||||
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
||||||
if e.beforePreparationErr != nil {
|
|
||||||
return nil, e.beforePreparationErr
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
if err := callback(promptexec.Preparation{
|
|
||||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
||||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
|
|
||||||
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
|
|
||||||
}, nil); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if e.afterPreparationErr != nil {
|
|
||||||
return nil, e.afterPreparationErr
|
|
||||||
}
|
|
||||||
validation := e.validation
|
|
||||||
if validation == "" {
|
|
||||||
validation = promptexec.ValidationPassed
|
|
||||||
}
|
|
||||||
return &promptexec.Execution{
|
|
||||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
||||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
||||||
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
|
|
||||||
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
|
|
||||||
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`),
|
|
||||||
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type successfulNotifier struct{}
|
|
||||||
|
|
||||||
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
|
||||||
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type failingNotifier struct{}
|
|
||||||
|
|
||||||
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
|
||||||
return nil, errors.New("injected notification failure")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
failOperation string
|
|
||||||
failMetadataCall int
|
|
||||||
output bool
|
|
||||||
notify bool
|
|
||||||
want reachedPromptArtifacts
|
|
||||||
}{
|
|
||||||
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
|
|
||||||
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
|
|
||||||
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
|
|
||||||
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
|
|
||||||
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
|
|
||||||
{name: "rendered report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
|
|
||||||
{name: "output then metadata", failOperation: failMetadata, failMetadataCall: 5, output: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
||||||
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
|
|
||||||
req.Store = store
|
|
||||||
if test.output {
|
|
||||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
||||||
paths.output = req.OutputPath
|
|
||||||
}
|
|
||||||
if test.notify {
|
|
||||||
req.Config.Notify.Distributor.Enabled = true
|
|
||||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
||||||
req.Notifier = successfulNotifier{}
|
|
||||||
req.noNotify = false
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := generatePromptReport(context.Background(), req)
|
|
||||||
if err == nil || result == nil {
|
|
||||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
||||||
}
|
|
||||||
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
executor artifactPathExecutor
|
|
||||||
want reachedPromptArtifacts
|
|
||||||
wantExecutionStatus state.PromptExecutionStatus
|
|
||||||
wantExecutionPaths state.PromptExecutionPaths
|
|
||||||
wantRawExecution bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "preparation failure",
|
|
||||||
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
|
|
||||||
want: reachedPromptArtifacts{preparation: true, metadata: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "operational execution failure",
|
|
||||||
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
|
|
||||||
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
|
|
||||||
wantExecutionStatus: state.PromptExecutionFailed,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "completed validation rejection",
|
|
||||||
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
|
|
||||||
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
|
||||||
wantExecutionStatus: state.PromptExecutionValidationRejected,
|
|
||||||
wantRawExecution: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
req, paths := promptArtifactRequest(t, test.executor)
|
|
||||||
if test.wantRawExecution {
|
|
||||||
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
|
|
||||||
}
|
|
||||||
result, err := generatePromptReport(context.Background(), req)
|
|
||||||
if err == nil || result == nil {
|
|
||||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
||||||
}
|
|
||||||
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
||||||
if test.wantExecutionStatus != "" {
|
|
||||||
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
|
|
||||||
if loadErr != nil {
|
|
||||||
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
|
|
||||||
}
|
|
||||||
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
|
|
||||||
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
|
|
||||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
||||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
||||||
paths.output = req.OutputPath
|
|
||||||
req.Config.Notify.Distributor.Enabled = true
|
|
||||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
||||||
req.Notifier = successfulNotifier{}
|
|
||||||
req.noNotify = false
|
|
||||||
|
|
||||||
result, err := generatePromptReport(context.Background(), req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("generatePromptReport() error = %v", err)
|
|
||||||
}
|
|
||||||
want := state.PromptExecutionPaths{
|
|
||||||
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
|
|
||||||
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
|
|
||||||
OutputPath: paths.output,
|
|
||||||
}
|
|
||||||
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
|
|
||||||
|
|
||||||
data, err := os.ReadFile(result.ExecutionPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read execution artifact: %v", err)
|
|
||||||
}
|
|
||||||
text := string(data)
|
|
||||||
for _, forbidden := range []string{
|
|
||||||
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
|
|
||||||
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
|
|
||||||
"credential", "secret-value",
|
|
||||||
} {
|
|
||||||
if strings.Contains(text, forbidden) {
|
|
||||||
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
failOperation string
|
|
||||||
failExecutionCall int
|
|
||||||
failMetadataCall int
|
|
||||||
requestOutput bool
|
|
||||||
failOutput bool
|
|
||||||
notify bool
|
|
||||||
notificationFailure bool
|
|
||||||
wantExecution reachedExecutionArtifacts
|
|
||||||
wantResult reachedPromptArtifacts
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "normalized text write", failOperation: failGeneratedText,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "render context write", failOperation: failRenderContext,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "rendered report write", failOperation: failRenderedReportPath,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "rendered report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "output write", requestOutput: true, failOutput: true,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "output checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "output metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
|
|
||||||
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
||||||
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
||||||
store := &failingPersistenceStore{
|
|
||||||
Store: req.Store, failOperation: test.failOperation,
|
|
||||||
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
|
|
||||||
}
|
|
||||||
if test.failOperation == failRenderedReportPath {
|
|
||||||
store.renderedReportPath = t.TempDir()
|
|
||||||
}
|
|
||||||
req.Store = store
|
|
||||||
if test.requestOutput {
|
|
||||||
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
||||||
paths.output = req.OutputPath
|
|
||||||
}
|
|
||||||
if test.failOutput {
|
|
||||||
blocker := filepath.Join(t.TempDir(), "not-a-directory")
|
|
||||||
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
|
|
||||||
t.Fatalf("write output blocker: %v", err)
|
|
||||||
}
|
|
||||||
req.OutputPath = filepath.Join(blocker, "daily.md")
|
|
||||||
paths.output = req.OutputPath
|
|
||||||
}
|
|
||||||
if test.notify {
|
|
||||||
req.Config.Notify.Distributor.Enabled = true
|
|
||||||
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
||||||
req.Notifier = successfulNotifier{}
|
|
||||||
req.noNotify = false
|
|
||||||
}
|
|
||||||
if test.notificationFailure {
|
|
||||||
req.Notifier = failingNotifier{}
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := generatePromptReport(context.Background(), req)
|
|
||||||
if err == nil || result == nil {
|
|
||||||
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
||||||
}
|
|
||||||
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
|
|
||||||
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type reachedExecutionArtifacts struct {
|
|
||||||
raw bool
|
|
||||||
normalized bool
|
|
||||||
renderContext bool
|
|
||||||
report bool
|
|
||||||
output bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
|
|
||||||
result := state.PromptExecutionPaths{}
|
|
||||||
if reached.raw {
|
|
||||||
result.RawOutputPath = paths.GeneratedTextRaw
|
|
||||||
}
|
|
||||||
if reached.normalized {
|
|
||||||
result.GeneratedTextPath = paths.GeneratedText
|
|
||||||
}
|
|
||||||
if reached.renderContext {
|
|
||||||
result.RenderContextPath = paths.RenderContext
|
|
||||||
}
|
|
||||||
if reached.report {
|
|
||||||
result.RenderedReportPath = paths.RenderedReport
|
|
||||||
}
|
|
||||||
if reached.output {
|
|
||||||
result.OutputPath = paths.output
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
|
|
||||||
t.Helper()
|
|
||||||
artifact, err := store.LoadPromptExecution(context.Background(), path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadPromptExecution() error = %v", err)
|
|
||||||
}
|
|
||||||
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
|
|
||||||
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
|
|
||||||
}
|
|
||||||
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
|
|
||||||
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
|
|
||||||
}
|
|
||||||
if artifact.Paths != want {
|
|
||||||
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptArtifactPaths struct {
|
|
||||||
state.ArtifactPaths
|
|
||||||
output string
|
|
||||||
}
|
|
||||||
|
|
||||||
type reachedPromptArtifacts struct {
|
|
||||||
preparation bool
|
|
||||||
execution bool
|
|
||||||
metadata bool
|
|
||||||
raw bool
|
|
||||||
normalized bool
|
|
||||||
renderContext bool
|
|
||||||
report bool
|
|
||||||
output bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
|
|
||||||
t.Helper()
|
|
||||||
cfg := config.Defaults()
|
|
||||||
cfg.Workspace.Root = t.TempDir()
|
|
||||||
resolved, err := ResolveGenerate(GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
|
|
||||||
}, mustParse("2026-05-29T05:00:00-05:00"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read daily fixture: %v", err)
|
|
||||||
}
|
|
||||||
var bundle weatherdata.Bundle
|
|
||||||
if err := json.Unmarshal(bundleData, &bundle); err != nil {
|
|
||||||
t.Fatalf("decode daily fixture: %v", err)
|
|
||||||
}
|
|
||||||
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
||||||
}
|
|
||||||
paths, err := filesystemStore.Paths(resolved)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Paths() error = %v", err)
|
|
||||||
}
|
|
||||||
debugWriter, err := promptdebug.NewPromptDebugWriter("")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
||||||
}
|
|
||||||
return promptReportRequest{
|
|
||||||
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
|
|
||||||
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
|
|
||||||
Inspection: PromptInspectionResult{
|
|
||||||
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
|
||||||
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
|
|
||||||
},
|
|
||||||
DebugWriter: debugWriter, noNotify: true,
|
|
||||||
}, promptArtifactPaths{ArtifactPaths: paths}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
|
|
||||||
t.Helper()
|
|
||||||
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
|
|
||||||
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
|
|
||||||
}
|
|
||||||
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
|
|
||||||
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
|
|
||||||
}
|
|
||||||
checks := []struct {
|
|
||||||
name string
|
|
||||||
got string
|
|
||||||
metadataGot string
|
|
||||||
inMetadata bool
|
|
||||||
path string
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
|
|
||||||
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
|
|
||||||
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
|
|
||||||
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
|
|
||||||
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
|
|
||||||
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
|
|
||||||
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
|
|
||||||
{"output", result.OutputPath, "", false, paths.output, want.output},
|
|
||||||
}
|
|
||||||
for _, check := range checks {
|
|
||||||
if check.want && check.got != check.path {
|
|
||||||
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
|
|
||||||
}
|
|
||||||
if check.want && check.inMetadata && check.metadataGot != check.path {
|
|
||||||
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
|
|
||||||
}
|
|
||||||
if !check.want && check.got != "" {
|
|
||||||
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
|
|
||||||
}
|
|
||||||
if !check.want && check.inMetadata && check.metadataGot != "" {
|
|
||||||
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,6 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||||
@@ -14,7 +13,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptReportRequest struct {
|
type promptReportRequest struct {
|
||||||
@@ -23,9 +22,23 @@ type promptReportRequest struct {
|
|||||||
Collection collect.Result
|
Collection collect.Result
|
||||||
Inspection PromptInspectionResult
|
Inspection PromptInspectionResult
|
||||||
DebugWriter *promptdebug.PromptDebugWriter
|
DebugWriter *promptdebug.PromptDebugWriter
|
||||||
|
Result *ReportResult
|
||||||
noNotify bool
|
noNotify bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type promptReportWorkflow struct {
|
||||||
|
ctx context.Context
|
||||||
|
req promptReportRequest
|
||||||
|
result *ReportResult
|
||||||
|
briefingMetadata briefing.Metadata
|
||||||
|
reportFacts ReportFacts
|
||||||
|
moduleSnapshot module.Snapshot
|
||||||
|
dataPackage []byte
|
||||||
|
handler generatedtext.Handler
|
||||||
|
debugRef promptdebug.PromptDebugRef
|
||||||
|
callbackFailed bool
|
||||||
|
}
|
||||||
|
|
||||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||||
workflow, err := newPromptReportWorkflow(ctx, req)
|
workflow, err := newPromptReportWorkflow(ctx, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -34,369 +47,145 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
|
|||||||
if err := workflow.buildInputs(); err != nil {
|
if err := workflow.buildInputs(); err != nil {
|
||||||
return workflow.result, err
|
return workflow.result, err
|
||||||
}
|
}
|
||||||
|
execution, err := workflow.executePrompt()
|
||||||
execution, executeErr := workflow.executePrompt()
|
if err != nil {
|
||||||
if executeErr != nil {
|
if workflow.callbackFailed {
|
||||||
return workflow.result, workflow.handleExecutionFailure(executeErr)
|
return workflow.result, err
|
||||||
|
}
|
||||||
|
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
|
||||||
}
|
}
|
||||||
if execution == nil {
|
if execution == nil {
|
||||||
err := promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)
|
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
|
||||||
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
|
||||||
return workflow.result, saveErr
|
|
||||||
}
|
}
|
||||||
return workflow.result, workflow.reportError("execute prompt", err)
|
workflow.result.ValidationStatus = execution.Validation.Status
|
||||||
}
|
if err := workflow.writeExecutionDebug(*execution); err != nil {
|
||||||
if err := workflow.persistExecutionDebug(*execution); err != nil {
|
|
||||||
return workflow.result, err
|
return workflow.result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
|
||||||
err := promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)
|
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
|
||||||
if saveErr := workflow.persistOperationalExecutionFailure(err); saveErr != nil {
|
|
||||||
return workflow.result, saveErr
|
|
||||||
}
|
|
||||||
return workflow.result, workflow.reportError("validate prompt execution", err)
|
|
||||||
}
|
|
||||||
if err := workflow.persistCompletedExecution(*execution); err != nil {
|
|
||||||
return workflow.result, err
|
|
||||||
}
|
}
|
||||||
if execution.Validation.Status == promptexec.ValidationFailed {
|
if execution.Validation.Status == promptexec.ValidationFailed {
|
||||||
err := promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)
|
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
|
||||||
return workflow.result, workflow.reportError("validate prompt execution", err)
|
|
||||||
}
|
}
|
||||||
|
return workflow.renderAndPublish(execution.RawOutput)
|
||||||
rendered, err := workflow.persistGeneratedContent(execution.RawOutput)
|
|
||||||
if err != nil {
|
|
||||||
return workflow.result, err
|
|
||||||
}
|
|
||||||
return workflow.finalizeReport(rendered)
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptReportWorkflow struct {
|
|
||||||
ctx context.Context
|
|
||||||
req promptReportRequest
|
|
||||||
store state.Store
|
|
||||||
result *ReportResult
|
|
||||||
metadata state.Metadata
|
|
||||||
briefingMetadata briefing.Metadata
|
|
||||||
reportFacts ReportFacts
|
|
||||||
moduleSnapshot module.Snapshot
|
|
||||||
dataPackageBytes []byte
|
|
||||||
handler generatedtext.Handler
|
|
||||||
executionArtifact state.PromptExecutionArtifact
|
|
||||||
debugRef promptdebug.PromptDebugRef
|
|
||||||
prepared bool
|
|
||||||
callbackFailed bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
|
||||||
if req.Collection.Bundle == nil {
|
if req.Collection.Bundle == nil {
|
||||||
return nil, fmt.Errorf("collected weather bundle is required")
|
return nil, fmt.Errorf("collected weather bundle is required")
|
||||||
}
|
}
|
||||||
store := req.Store
|
result := req.Result
|
||||||
var err error
|
if result == nil {
|
||||||
if store == nil {
|
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
|
||||||
store, err = defaultStore(req.Config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
return &promptReportWorkflow{
|
||||||
|
ctx: ctx, req: req,
|
||||||
|
result: result,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
|
||||||
|
metadata := resolved.Metadata()
|
||||||
|
return &ReportResult{
|
||||||
|
ReportID: resolved.Definition.ID, ReportName: resolved.Definition.Name,
|
||||||
|
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
||||||
|
RunID: metadata.RunID, GeneratedAt: metadata.GeneratedAt, Timezone: req.Config.WeatherAPI.Timezone,
|
||||||
|
ValidPeriod: metadata.ValidPeriod,
|
||||||
|
ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName,
|
||||||
}
|
}
|
||||||
return &promptReportWorkflow{ctx: ctx, req: req, store: store}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) buildInputs() error {
|
func (w *promptReportWorkflow) buildInputs() error {
|
||||||
paths, err := w.store.Paths(w.req.Resolved)
|
var err error
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result = &ReportResult{}
|
|
||||||
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
w.reportFacts, err = BuildReportFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.req.Collection.Bundle)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build report facts", err)
|
return w.reportError("build report facts", err)
|
||||||
}
|
}
|
||||||
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
w.moduleSnapshot, err = BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{Config: w.req.Config, Resolved: w.req.Resolved}, w.reportFacts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return generatedReportError(w.req.Resolved, w.req.Resolved.Metadata().RunID, "build module snapshot", err)
|
return w.reportError("build module snapshot", err)
|
||||||
}
|
}
|
||||||
moduleSnapshotPath, err := w.store.SaveModuleSnapshot(w.ctx, w.req.Resolved, w.moduleSnapshot)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result.ModuleSnapshot = w.moduleSnapshot
|
|
||||||
w.result.ModuleSnapshotPath = moduleSnapshotPath
|
|
||||||
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
w.briefingMetadata = briefing.BuildMetadata(briefingBuildContext(w.req.Config, w.req.Resolved, w.reportFacts.Collected))
|
||||||
w.metadata = state.BuildPromptMetadataFromBriefingMetadata(w.req.Resolved, w.briefingMetadata, state.ArtifactPaths{
|
w.result.SourceWarnings = append([]weatherdata.SourceWarning(nil), w.briefingMetadata.SourceWarnings...)
|
||||||
ModuleSnapshot: moduleSnapshotPath,
|
dataPackage, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptMetadata(w.briefingMetadata), Modules: w.moduleSnapshot})
|
||||||
Metadata: paths.Metadata,
|
|
||||||
})
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
|
|
||||||
Metadata: promptMetadata(w.metadata), Modules: w.moduleSnapshot,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return w.reportError("build data package", err)
|
return w.reportError("build data package", err)
|
||||||
}
|
}
|
||||||
w.dataPackageBytes, err = promptinput.MarshalYAML(dataPackage)
|
w.dataPackage, err = promptinput.MarshalYAML(dataPackage)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return w.reportError("marshal data package", err)
|
||||||
}
|
}
|
||||||
dataPackagePath, err := w.store.SaveDataPackageBytes(w.ctx, w.req.Resolved, w.dataPackageBytes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.metadata.DataPackagePath = dataPackagePath
|
|
||||||
w.result.DataPackage = dataPackage
|
|
||||||
w.result.DataPackagePath = dataPackagePath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
w.handler, err = generatedtext.LookupDefinition(w.req.Resolved.Definition)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return w.reportError("lookup generated text catalog", err)
|
return w.reportError("lookup generated text catalog", err)
|
||||||
}
|
}
|
||||||
w.debugRef = promptdebug.PromptDebugRef{
|
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
|
||||||
ReportID: w.req.Resolved.Definition.ID, ValidDate: w.req.Resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.metadata.RunID,
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
|
||||||
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{
|
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
|
||||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackage, CaptureDebug: captureDebug}, w.writePreparationDebug)
|
||||||
ProfileID: w.req.Inspection.ProfileID, DataPackage: w.dataPackageBytes,
|
|
||||||
DataPackagePath: w.result.DataPackagePath, CaptureDebug: w.req.DebugWriter.Enabled(),
|
|
||||||
}, w.persistPreparation)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistPreparation(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
|
||||||
artifact := state.PromptPreparationArtifact{
|
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
|
||||||
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationSucceeded,
|
if w.req.DebugWriter == nil {
|
||||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
return nil
|
||||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
|
||||||
DataPackagePath: w.result.DataPackagePath, Preparation: &preparation,
|
|
||||||
StartedAt: preparation.StartedAt, EndedAt: preparation.EndedAt, Duration: preparation.Duration,
|
|
||||||
}
|
}
|
||||||
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
||||||
if err != nil {
|
|
||||||
w.callbackFailed = true
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.prepared = true
|
|
||||||
w.result.PreparationPath = path
|
|
||||||
w.metadata.PreparationPath = path
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
debugPath, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.callbackFailed = true
|
w.callbackFailed = true
|
||||||
return promptDebugWriteError(err)
|
return promptDebugWriteError(err)
|
||||||
}
|
}
|
||||||
if debugPath != "" {
|
w.result.LLMDebugPath = path
|
||||||
w.result.LLMDebugPath = debugPath
|
|
||||||
}
|
|
||||||
if err := w.saveMetadata(); err != nil {
|
|
||||||
w.callbackFailed = true
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) handleExecutionFailure(executeErr error) error {
|
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
|
||||||
if w.callbackFailed {
|
if w.req.DebugWriter == nil {
|
||||||
return executeErr
|
return nil
|
||||||
}
|
}
|
||||||
executeErr = classifiedPromptError("prompt execution failed", executeErr)
|
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
||||||
if !w.prepared {
|
|
||||||
if err := w.persistPreparationFailure(executeErr); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return w.reportError("prepare prompt", executeErr)
|
|
||||||
}
|
|
||||||
if promptexec.CategoryOf(executeErr) != "" {
|
|
||||||
if err := w.persistOperationalExecutionFailure(executeErr); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return w.reportError("execute prompt", executeErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistPreparationFailure(executeErr error) error {
|
|
||||||
startedAt, endedAt := time.Now(), time.Now()
|
|
||||||
artifact := state.PromptPreparationArtifact{
|
|
||||||
SchemaVersion: state.PromptPreparationSchemaVersion, Status: state.PromptPreparationFailed,
|
|
||||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
|
||||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
|
||||||
DataPackagePath: w.result.DataPackagePath, StartedAt: startedAt, EndedAt: endedAt,
|
|
||||||
Error: state.NewPromptArtifactError(executeErr),
|
|
||||||
}
|
|
||||||
path, err := w.store.SavePromptPreparation(w.ctx, w.req.Resolved, artifact)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result.PreparationPath = path
|
|
||||||
w.metadata.PreparationPath = path
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
return w.saveMetadata()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistOperationalExecutionFailure(executeErr error) error {
|
|
||||||
artifact := failedPromptExecutionArtifact(w.req.Resolved, w.metadata, w.req.Inspection, executeErr)
|
|
||||||
path, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, artifact)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result.ExecutionPath = path
|
|
||||||
w.metadata.ExecutionPath = path
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
return w.saveMetadata()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistExecutionDebug(execution promptexec.Execution) error {
|
|
||||||
debugPath, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
return w.reportError("write prompt debug", promptDebugWriteError(err))
|
||||||
}
|
}
|
||||||
if debugPath != "" {
|
if path != "" {
|
||||||
w.result.LLMDebugPath = debugPath
|
w.result.LLMDebugPath = path
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistCompletedExecution(execution promptexec.Execution) error {
|
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
|
||||||
rawPath, err := w.store.SaveGeneratedTextRaw(w.ctx, w.req.Resolved, execution.RawOutput)
|
generatedText, _, err := w.handler.Validate(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return w.result, w.reportError("validate generated text", err)
|
||||||
}
|
}
|
||||||
w.result.GeneratedTextRawPath = rawPath
|
|
||||||
w.metadata.GeneratedTextRawPath = rawPath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
w.executionArtifact = state.PromptExecutionArtifact{
|
|
||||||
SchemaVersion: state.PromptExecutionSchemaVersion,
|
|
||||||
ReportID: w.req.Resolved.Definition.ID, RunID: w.metadata.RunID,
|
|
||||||
PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion,
|
|
||||||
Provenance: ptr(state.PromptExecutionProvenanceFrom(execution)), Validation: &execution.Validation,
|
|
||||||
Paths: state.PromptExecutionPaths{RawOutputPath: rawPath},
|
|
||||||
StartedAt: execution.StartedAt, EndedAt: execution.EndedAt, Duration: execution.Duration,
|
|
||||||
}
|
|
||||||
if execution.Validation.Status == promptexec.ValidationPassed {
|
|
||||||
w.executionArtifact.Status = state.PromptExecutionSucceeded
|
|
||||||
} else {
|
|
||||||
w.executionArtifact.Status = state.PromptExecutionValidationRejected
|
|
||||||
}
|
|
||||||
executionPath, err := w.store.SavePromptExecution(w.ctx, w.req.Resolved, w.executionArtifact)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result.ExecutionPath = executionPath
|
|
||||||
w.metadata.ExecutionPath = executionPath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
return w.saveMetadata()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistGeneratedContent(raw []byte) ([]byte, error) {
|
|
||||||
generatedText, normalized, err := w.handler.Validate(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, w.reportError("validate generated text", err)
|
|
||||||
}
|
|
||||||
generatedTextPath, err := w.store.SaveGeneratedText(w.ctx, w.req.Resolved, normalized)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
w.result.GeneratedTextPath = generatedTextPath
|
|
||||||
w.metadata.GeneratedTextPath = generatedTextPath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.GeneratedTextPath = generatedTextPath }); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
renderContext, err := w.handler.BuildRenderContext(w.briefingMetadata, w.moduleSnapshot, w.reportFacts.Collected, w.reportFacts.Derived, generatedText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, w.reportError("build render context", err)
|
return w.result, w.reportError("build render context", err)
|
||||||
}
|
}
|
||||||
renderContextPath, err := w.store.SaveRenderContext(w.ctx, w.req.Resolved, renderContext)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
w.result.RenderContextPath = renderContextPath
|
|
||||||
w.metadata.RenderContextPath = renderContextPath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
if err := w.persistReachedPathAndMetadata(func(paths *state.PromptExecutionPaths) { paths.RenderContextPath = renderContextPath }); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
rendered, err := w.handler.Render(renderContext)
|
rendered, err := w.handler.Render(renderContext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, w.reportError("render template", err)
|
return w.result, w.reportError("render template", err)
|
||||||
}
|
}
|
||||||
return rendered, nil
|
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
|
||||||
|
return w.result, err
|
||||||
}
|
}
|
||||||
|
w.result.OutputPath = w.req.OutputPath
|
||||||
func (w *promptReportWorkflow) finalizeReport(rendered []byte) (*ReportResult, error) {
|
if w.req.noNotify {
|
||||||
reportPath, err := w.store.PrepareRenderedReport(w.ctx, w.req.Resolved)
|
return w.result, nil
|
||||||
|
}
|
||||||
|
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
|
||||||
|
w.result.Notification = notification
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return w.result, err
|
return w.result, err
|
||||||
}
|
}
|
||||||
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
|
return w.result, nil
|
||||||
return w.result, err
|
|
||||||
}
|
|
||||||
w.result.ReportPath = reportPath
|
|
||||||
w.metadata.RenderedReportPath = reportPath
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
if err := w.persistReachedPath(func(paths *state.PromptExecutionPaths) { paths.RenderedReportPath = reportPath }); err != nil {
|
|
||||||
return w.result, err
|
|
||||||
}
|
|
||||||
finalized, err := finalizeRenderedReport(w.ctx, finalizeRenderedReportRequest{
|
|
||||||
Config: w.req.Config, Store: w.store, Resolved: w.req.Resolved, Metadata: w.metadata, MetadataPath: w.result.MetadataPath,
|
|
||||||
ExecutionArtifact: &w.executionArtifact, RenderedReportPath: reportPath, OutputPath: w.req.OutputPath,
|
|
||||||
Notifier: w.req.Notifier, noNotify: w.req.noNotify,
|
|
||||||
})
|
|
||||||
w.result.OutputPath = finalized.OutputPath
|
|
||||||
w.result.Metadata, w.result.MetadataPath, w.result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
|
|
||||||
return w.result, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistReachedPath(update func(*state.PromptExecutionPaths)) error {
|
|
||||||
return persistReachedPromptPath(w.ctx, w.store, w.req.Resolved, &w.executionArtifact, update)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) persistReachedPathAndMetadata(update func(*state.PromptExecutionPaths)) error {
|
|
||||||
if err := w.persistReachedPath(update); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return w.saveMetadata()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *promptReportWorkflow) saveMetadata() error {
|
|
||||||
path, err := w.store.SaveMetadata(w.ctx, w.metadata)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
w.result.Metadata = w.metadata
|
|
||||||
w.result.MetadataPath = path
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
func (w *promptReportWorkflow) reportError(operation string, err error) error {
|
||||||
return generatedReportError(w.req.Resolved, w.metadata.RunID, operation, err)
|
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
|
||||||
}
|
|
||||||
|
|
||||||
func persistReachedPromptPath(
|
|
||||||
ctx context.Context,
|
|
||||||
store state.Store,
|
|
||||||
resolved report.Resolved,
|
|
||||||
artifact *state.PromptExecutionArtifact,
|
|
||||||
update func(*state.PromptExecutionPaths),
|
|
||||||
) error {
|
|
||||||
update(&artifact.Paths)
|
|
||||||
_, err := store.SavePromptExecution(ctx, resolved, *artifact)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func failedPromptExecutionArtifact(resolved report.Resolved, metadata state.Metadata, inspection PromptInspectionResult, err error) state.PromptExecutionArtifact {
|
|
||||||
now := time.Now()
|
|
||||||
return state.PromptExecutionArtifact{
|
|
||||||
SchemaVersion: state.PromptExecutionSchemaVersion, Status: state.PromptExecutionFailed,
|
|
||||||
ReportID: resolved.Definition.ID, RunID: metadata.RunID, PromptID: inspection.PromptID,
|
|
||||||
PromptVersion: inspection.PromptVersion, StartedAt: now, EndedAt: now,
|
|
||||||
Error: state.NewPromptArtifactError(err),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func classifiedPromptError(operation string, err error) error {
|
func classifiedPromptError(operation string, err error) error {
|
||||||
@@ -409,5 +198,3 @@ func classifiedPromptError(operation string, err error) error {
|
|||||||
func promptDebugWriteError(err error) error {
|
func promptDebugWriteError(err error) error {
|
||||||
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
|
return promptexec.NewError(promptexec.InvalidConfiguration, "write requested prompt debug artifact", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func ptr[T any](value T) *T { return &value }
|
|
||||||
|
|||||||
@@ -1,671 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
||||||
)
|
|
||||||
|
|
||||||
type workflowCollector struct {
|
|
||||||
result *collect.Result
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
|
||||||
c.calls++
|
|
||||||
if c.err != nil {
|
|
||||||
return nil, c.err
|
|
||||||
}
|
|
||||||
return c.result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type workflowExecutor struct {
|
|
||||||
definition report.Definition
|
|
||||||
raw []byte
|
|
||||||
prompt promptexec.PromptInspection
|
|
||||||
inspectionErr error
|
|
||||||
profile promptexec.ProfileInspection
|
|
||||||
profileErr error
|
|
||||||
beforePreparationErr error
|
|
||||||
afterCallbackErr error
|
|
||||||
afterPreparationErr error
|
|
||||||
validation promptexec.ValidationStatus
|
|
||||||
executeCalls int
|
|
||||||
providerCalls int
|
|
||||||
request promptexec.ExecuteRequest
|
|
||||||
beforeProvider func()
|
|
||||||
preparationDebug *promptexec.PreparationDebug
|
|
||||||
executionDebug *promptexec.ExecutionDebug
|
|
||||||
preparation *promptexec.Preparation
|
|
||||||
execution *promptexec.Execution
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
|
||||||
if e.inspectionErr != nil {
|
|
||||||
return promptexec.PromptInspection{}, e.inspectionErr
|
|
||||||
}
|
|
||||||
if id != e.definition.PromptID || version != e.definition.PromptVersion {
|
|
||||||
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
|
|
||||||
}
|
|
||||||
if e.prompt.PromptID != "" {
|
|
||||||
return e.prompt, nil
|
|
||||||
}
|
|
||||||
return validPromptInspection(e.definition), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
||||||
if e.profileErr != nil {
|
|
||||||
return promptexec.ProfileInspection{}, e.profileErr
|
|
||||||
}
|
|
||||||
profile := e.profile
|
|
||||||
if profile.ProfileID == "" {
|
|
||||||
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
|
|
||||||
}
|
|
||||||
return profile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
||||||
e.executeCalls++
|
|
||||||
e.request = req
|
|
||||||
if e.beforePreparationErr != nil {
|
|
||||||
return nil, e.beforePreparationErr
|
|
||||||
}
|
|
||||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
profile := e.profile
|
|
||||||
if profile.ProfileID == "" {
|
|
||||||
profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"}
|
|
||||||
}
|
|
||||||
preparation := promptexec.Preparation{
|
|
||||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
||||||
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
|
|
||||||
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
|
|
||||||
}
|
|
||||||
e.preparation = &preparation
|
|
||||||
if err := callback(preparation, e.preparationDebug); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if e.afterCallbackErr != nil {
|
|
||||||
return nil, e.afterCallbackErr
|
|
||||||
}
|
|
||||||
if e.beforeProvider != nil {
|
|
||||||
e.beforeProvider()
|
|
||||||
}
|
|
||||||
e.providerCalls++
|
|
||||||
if e.afterPreparationErr != nil {
|
|
||||||
return nil, e.afterPreparationErr
|
|
||||||
}
|
|
||||||
validation := e.validation
|
|
||||||
if validation == "" {
|
|
||||||
validation = promptexec.ValidationPassed
|
|
||||||
}
|
|
||||||
execution := &promptexec.Execution{
|
|
||||||
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
||||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
||||||
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
|
|
||||||
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
|
|
||||||
Debug: e.executionDebug,
|
|
||||||
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
|
||||||
}
|
|
||||||
e.execution = execution
|
|
||||||
return execution, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type workflowNotifier struct {
|
|
||||||
requests []NotificationRequest
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *workflowNotifier) Notify(_ context.Context, req NotificationRequest) (*NotificationResult, error) {
|
|
||||||
n.requests = append(n.requests, req)
|
|
||||||
if n.err != nil {
|
|
||||||
return nil, n.err
|
|
||||||
}
|
|
||||||
return &NotificationResult{
|
|
||||||
RunID: "notification-run", PipelineID: req.PipelineID, BundleID: req.BundleID,
|
|
||||||
IdempotencyKey: req.IdempotencyKey, Status: "succeeded", UploadStatus: "accepted",
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
kind ReportKind
|
|
||||||
id report.ID
|
|
||||||
date time.Time
|
|
||||||
raw string
|
|
||||||
wantOutput string
|
|
||||||
}{
|
|
||||||
{name: "daily", kind: ReportDaily, id: report.Daily, date: workflowTime("2026-05-29T12:00:00-05:00"), raw: validDailyWorkflowJSON(), wantOutput: "Showers are possible during the selected day."},
|
|
||||||
{name: "today", kind: ReportToday, id: report.Today, raw: validTodayWorkflowJSON(), wantOutput: "Today starts with showers before improving."},
|
|
||||||
{name: "tomorrow", kind: ReportTomorrow, id: report.Tomorrow, raw: validTomorrowWorkflowJSON(), wantOutput: "Tomorrow starts with showers before improving."},
|
|
||||||
{name: "hourly", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), wantOutput: "Storm chances increase through late morning."},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
definition := report.DefaultRegistry().MustLookup(test.id)
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
|
||||||
notifier := &workflowNotifier{}
|
|
||||||
outputPath := filepath.Join(t.TempDir(), test.name+".md")
|
|
||||||
|
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: test.kind, Date: test.date, Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
OutputPath: outputPath, Collector: collector, Executor: executor, Notifier: notifier,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if result.Metadata.ReportID != test.id || result.Metadata.PromptID != definition.PromptID {
|
|
||||||
t.Fatalf("metadata identity = %q/%q, want %q/%q", result.Metadata.ReportID, result.Metadata.PromptID, test.id, definition.PromptID)
|
|
||||||
}
|
|
||||||
if executor.request.PromptVersion != definition.PromptVersion {
|
|
||||||
t.Fatalf("prompt version = %q, want %q", executor.request.PromptVersion, definition.PromptVersion)
|
|
||||||
}
|
|
||||||
filesystem, storeErr := state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
if storeErr != nil {
|
|
||||||
t.Fatalf("NewFilesystemStore() error = %v", storeErr)
|
|
||||||
}
|
|
||||||
preparation, loadErr := filesystem.LoadPromptPreparation(context.Background(), result.PreparationPath)
|
|
||||||
if loadErr != nil || preparation.PromptVersion != definition.PromptVersion {
|
|
||||||
t.Fatalf("persisted preparation prompt version = %q, error %v, want %q", preparation.PromptVersion, loadErr, definition.PromptVersion)
|
|
||||||
}
|
|
||||||
if collector.calls != 1 || executor.executeCalls != 1 || executor.providerCalls != 1 {
|
|
||||||
t.Fatalf("calls = collect %d execute %d provider %d, want one each", collector.calls, executor.executeCalls, executor.providerCalls)
|
|
||||||
}
|
|
||||||
persisted, readErr := os.ReadFile(result.DataPackagePath)
|
|
||||||
if readErr != nil {
|
|
||||||
t.Fatalf("read data package: %v", readErr)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(executor.request.DataPackage, persisted) {
|
|
||||||
t.Fatal("executor data package differs from exact persisted YAML bytes")
|
|
||||||
}
|
|
||||||
renderedReport, readErr := os.ReadFile(result.ReportPath)
|
|
||||||
if readErr != nil || !strings.Contains(string(renderedReport), test.wantOutput) {
|
|
||||||
t.Fatalf("rendered report = %q, error %v, want generated template output %q", renderedReport, readErr, test.wantOutput)
|
|
||||||
}
|
|
||||||
copied, readErr := os.ReadFile(outputPath)
|
|
||||||
if readErr != nil || !bytes.Equal(copied, renderedReport) || result.OutputPath != outputPath {
|
|
||||||
t.Fatalf("output mismatch/error/path = %v/%q", readErr, result.OutputPath)
|
|
||||||
}
|
|
||||||
if len(notifier.requests) != 1 || notifier.requests[0].ReportPath != outputPath {
|
|
||||||
t.Fatalf("notification requests = %#v, want selected output source", notifier.requests)
|
|
||||||
}
|
|
||||||
wantPipeline := "reports." + string(test.id) + "." + definition.ArtifactGroup
|
|
||||||
if notifier.requests[0].PipelineID != wantPipeline {
|
|
||||||
t.Fatalf("pipeline = %q, want %q", notifier.requests[0].PipelineID, wantPipeline)
|
|
||||||
}
|
|
||||||
validDate := result.Metadata.ValidPeriod.Start.Format("2006-01-02")
|
|
||||||
wantBundlePaths := workflowBundlePaths(test.id, validDate, result.Metadata.RunID)
|
|
||||||
if strings.Join(notifier.requests[0].BundlePaths, "\n") != strings.Join(wantBundlePaths, "\n") {
|
|
||||||
t.Fatalf("bundle paths = %#v, want %#v", notifier.requests[0].BundlePaths, wantBundlePaths)
|
|
||||||
}
|
|
||||||
renderedName := filepath.Base(result.ReportPath)
|
|
||||||
if !strings.HasPrefix(renderedName, "report.") || !strings.Contains(renderedName, "_"+test.name) || !strings.HasSuffix(renderedName, ".md") || filepath.Base(result.OutputPath) != test.name+".md" {
|
|
||||||
t.Fatalf("output names = rendered %q copy %q", result.ReportPath, result.OutputPath)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
kind ReportKind
|
|
||||||
id report.ID
|
|
||||||
raw string
|
|
||||||
override string
|
|
||||||
profile promptexec.ProfileInspection
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(),
|
|
||||||
profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(),
|
|
||||||
profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile",
|
|
||||||
profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Promptkit.Profile = test.override
|
|
||||||
definition := report.DefaultRegistry().MustLookup(test.id)
|
|
||||||
executor := &workflowExecutor{
|
|
||||||
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
|
|
||||||
}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
_, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GenerateDetailed() error = %v", err)
|
|
||||||
}
|
|
||||||
if executor.request.ProfileID != test.profile.ProfileID {
|
|
||||||
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
|
|
||||||
}
|
|
||||||
if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName {
|
|
||||||
t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
|
|
||||||
}
|
|
||||||
if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName {
|
|
||||||
t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type preparationFailingStore struct {
|
|
||||||
state.Store
|
|
||||||
}
|
|
||||||
|
|
||||||
type renderContextFailingStore struct {
|
|
||||||
state.Store
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s renderContextFailingStore) SaveModuleSnapshot(ctx context.Context, resolved report.Resolved, snapshot module.Snapshot) (string, error) {
|
|
||||||
path, err := s.Store.SaveModuleSnapshot(ctx, resolved, snapshot)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for index := range snapshot.Outputs {
|
|
||||||
snapshot.Outputs[index].Value = "invalid module value"
|
|
||||||
}
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s preparationFailingStore) SavePromptPreparation(context.Context, report.Resolved, state.PromptPreparationArtifact) (string, error) {
|
|
||||||
return "", errors.New("injected preparation persistence failure")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedStopsAtConsequentialPromptFailures(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
configure func(*workflowExecutor)
|
|
||||||
wantCategory promptexec.ErrorCategory
|
|
||||||
wantPreparation bool
|
|
||||||
wantExecution bool
|
|
||||||
wantRaw bool
|
|
||||||
wantProviderCall int
|
|
||||||
}{
|
|
||||||
{name: "preparation", configure: func(e *workflowExecutor) {
|
|
||||||
e.beforePreparationErr = promptexec.NewError(promptexec.Generation, "preparation failed", nil)
|
|
||||||
}, wantCategory: promptexec.Generation, wantPreparation: true},
|
|
||||||
{name: "credential disappears", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterCallbackErr = promptexec.NewError(promptexec.MissingCredential, "credential unavailable", nil)
|
|
||||||
}, wantCategory: promptexec.MissingCredential, wantPreparation: true, wantExecution: true},
|
|
||||||
{name: "capacity is not retried", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterPreparationErr = promptexec.NewError(promptexec.Capacity, "capacity rejected", nil)
|
|
||||||
}, wantCategory: promptexec.Capacity, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "canceled", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterPreparationErr = promptexec.NewError(promptexec.Canceled, "request canceled", context.Canceled)
|
|
||||||
}, wantCategory: promptexec.Canceled, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "deadline", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterPreparationErr = promptexec.NewError(promptexec.DeadlineExceeded, "deadline exceeded", context.DeadlineExceeded)
|
|
||||||
}, wantCategory: promptexec.DeadlineExceeded, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "generation", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterPreparationErr = promptexec.NewError(promptexec.Generation, "generation failed", nil)
|
|
||||||
}, wantCategory: promptexec.Generation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "operational validation error", configure: func(e *workflowExecutor) {
|
|
||||||
e.afterPreparationErr = promptexec.NewError(promptexec.OperationalValidation, "validator failed", nil)
|
|
||||||
}, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "operational validation incomplete", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationSkipped }, wantCategory: promptexec.OperationalValidation, wantPreparation: true, wantExecution: true, wantProviderCall: 1},
|
|
||||||
{name: "schema rejection", configure: func(e *workflowExecutor) { e.validation = promptexec.ValidationFailed }, wantCategory: promptexec.ValidationRejected, wantPreparation: true, wantExecution: true, wantRaw: true, wantProviderCall: 1},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = false
|
|
||||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
||||||
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
|
||||||
test.configure(executor)
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
collector := &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: collector, Executor: executor,
|
|
||||||
})
|
|
||||||
if err == nil || result == nil || promptexec.CategoryOf(err) != test.wantCategory {
|
|
||||||
t.Fatalf("result/error/category = %#v/%v/%q, want partial result and %q", result, err, promptexec.CategoryOf(err), test.wantCategory)
|
|
||||||
}
|
|
||||||
if (result.PreparationPath != "") != test.wantPreparation || (result.ExecutionPath != "") != test.wantExecution || (result.GeneratedTextRawPath != "") != test.wantRaw {
|
|
||||||
t.Fatalf("paths = preparation %q execution %q raw %q", result.PreparationPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
|
||||||
}
|
|
||||||
if executor.executeCalls != 1 || executor.providerCalls != test.wantProviderCall {
|
|
||||||
t.Fatalf("calls = execute %d provider %d, want 1/%d", executor.executeCalls, executor.providerCalls, test.wantProviderCall)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
configure func(*workflowExecutor)
|
|
||||||
wantCategory promptexec.ErrorCategory
|
|
||||||
}{
|
|
||||||
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
|
|
||||||
{name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration},
|
|
||||||
{name: "malformed profile", configure: func(e *workflowExecutor) {
|
|
||||||
e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret")
|
|
||||||
}, wantCategory: promptexec.InvalidConfiguration},
|
|
||||||
{name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration},
|
|
||||||
{name: "credential", configure: func(e *workflowExecutor) {
|
|
||||||
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
|
|
||||||
}, wantCategory: promptexec.MissingCredential},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
||||||
executor := &workflowExecutor{definition: definition}
|
|
||||||
test.configure(executor)
|
|
||||||
collector := &workflowCollector{err: errors.New("collector must not run")}
|
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: collector, Executor: executor,
|
|
||||||
})
|
|
||||||
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
|
|
||||||
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") {
|
|
||||||
t.Fatalf("error leaks profile details: %v", err)
|
|
||||||
}
|
|
||||||
entries, readErr := os.ReadDir(cfg.Workspace.Root)
|
|
||||||
if readErr != nil || len(entries) != 0 {
|
|
||||||
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedStopsProviderWhenPreparationCannotPersist(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = false
|
|
||||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
||||||
}
|
|
||||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
||||||
executor := &workflowExecutor{definition: definition, raw: []byte(validDailyWorkflowJSON())}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Store: preparationFailingStore{Store: filesystem},
|
|
||||||
})
|
|
||||||
if err == nil || result == nil || result.PreparationPath != "" || executor.providerCalls != 0 {
|
|
||||||
t.Fatalf("result/error/preparation/provider = %#v/%v/%q/%d", result, err, result.PreparationPath, executor.providerCalls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedPersistsPreparationBeforeProviderExecution(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = false
|
|
||||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
|
||||||
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
|
|
||||||
resolved, err := ResolveGenerate(request, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
||||||
}
|
|
||||||
paths, err := filesystem.Paths(resolved)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Paths() error = %v", err)
|
|
||||||
}
|
|
||||||
checked := false
|
|
||||||
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
|
||||||
executor.beforeProvider = func() {
|
|
||||||
checked = true
|
|
||||||
if _, statErr := os.Stat(paths.Preparation); statErr != nil {
|
|
||||||
t.Fatalf("preparation was not durable before provider execution: %v", statErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
request.Executor = executor
|
|
||||||
request.Store = filesystem
|
|
||||||
result, err := GenerateDetailed(context.Background(), request)
|
|
||||||
if err != nil || result == nil || !checked || result.OutputPath == "" {
|
|
||||||
t.Fatalf("result/error/checked/output = %#v/%v/%t/%q", result, err, checked, result.OutputPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedRetainsInspectableArtifactsAfterApplicationFailures(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
raw string
|
|
||||||
configure func(*GenerateRequest, *workflowNotifier)
|
|
||||||
wantRaw bool
|
|
||||||
wantNormalized bool
|
|
||||||
wantContext bool
|
|
||||||
wantReport bool
|
|
||||||
wantOutput bool
|
|
||||||
}{
|
|
||||||
{name: "generated text decode", raw: `{`, wantRaw: true},
|
|
||||||
{name: "generated text domain", raw: `{}`, wantRaw: true},
|
|
||||||
{name: "render context build", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
||||||
req.Store = renderContextFailingStore{Store: req.Store}
|
|
||||||
}, wantRaw: true, wantNormalized: true},
|
|
||||||
{name: "render context persistence", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
||||||
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderContext}
|
|
||||||
}, wantRaw: true, wantNormalized: true},
|
|
||||||
{name: "template write", raw: validDailyWorkflowJSON(), configure: func(req *GenerateRequest, _ *workflowNotifier) {
|
|
||||||
blocker := filepath.Join(t.TempDir(), "report-blocker")
|
|
||||||
if err := os.Mkdir(blocker, 0o700); err != nil {
|
|
||||||
t.Fatalf("create report blocker: %v", err)
|
|
||||||
}
|
|
||||||
req.Store = &failingPersistenceStore{Store: req.Store, failOperation: failRenderedReportPath, renderedReportPath: blocker}
|
|
||||||
}, wantRaw: true, wantNormalized: true, wantContext: true},
|
|
||||||
{name: "notification", raw: validDailyWorkflowJSON(), configure: func(_ *GenerateRequest, notifier *workflowNotifier) {
|
|
||||||
notifier.err = errors.New("notification rejected")
|
|
||||||
}, wantRaw: true, wantNormalized: true, wantContext: true, wantReport: true, wantOutput: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
definition := report.DefaultRegistry().MustLookup(report.Daily)
|
|
||||||
executor := &workflowExecutor{definition: definition, raw: []byte(test.raw)}
|
|
||||||
notifier := &workflowNotifier{}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
filesystem, err := state.NewFilesystemStore(cfg.Workspace)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
||||||
}
|
|
||||||
req := GenerateRequest{
|
|
||||||
Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(),
|
|
||||||
OutputPath: filepath.Join(t.TempDir(), "daily.md"), Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}},
|
|
||||||
Executor: executor, Notifier: notifier, Store: filesystem,
|
|
||||||
}
|
|
||||||
if test.configure != nil {
|
|
||||||
test.configure(&req, notifier)
|
|
||||||
}
|
|
||||||
result, err := GenerateDetailed(context.Background(), req)
|
|
||||||
if err == nil || result == nil {
|
|
||||||
t.Fatalf("result/error = %#v/%v, want partial result and error", result, err)
|
|
||||||
}
|
|
||||||
if (result.GeneratedTextRawPath != "") != test.wantRaw || (result.GeneratedTextPath != "") != test.wantNormalized ||
|
|
||||||
(result.RenderContextPath != "") != test.wantContext || (result.ReportPath != "") != test.wantReport ||
|
|
||||||
(result.OutputPath != "") != test.wantOutput {
|
|
||||||
t.Fatalf("reached paths = raw %q normalized %q context %q report %q output %q", result.GeneratedTextRawPath, result.GeneratedTextPath, result.RenderContextPath, result.ReportPath, result.OutputPath)
|
|
||||||
}
|
|
||||||
if test.wantRaw {
|
|
||||||
persisted, readErr := os.ReadFile(result.GeneratedTextRawPath)
|
|
||||||
if readErr != nil || !bytes.Equal(persisted, []byte(test.raw)) {
|
|
||||||
t.Fatalf("retained raw output = %q, error %v", persisted, readErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if test.name == "notification" && (len(notifier.requests) != 1 || notifier.requests[0].ReportPath != result.OutputPath) {
|
|
||||||
t.Fatalf("notification requests = %#v", notifier.requests)
|
|
||||||
}
|
|
||||||
if test.name == "notification" && result.Metadata.NotificationPath != "" {
|
|
||||||
t.Fatalf("notification metadata retains a receipt path: %#v", result.Metadata)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateDetailedDebugFailuresRespectProviderBoundary(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
createCollision func(string, report.Resolved) error
|
|
||||||
wantProviderCalls int
|
|
||||||
wantPreparationFile bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "preparation debug",
|
|
||||||
createCollision: func(root string, resolved report.Resolved) error {
|
|
||||||
path := workflowDebugRunPath(root, resolved)
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return os.WriteFile(path, []byte("not a directory"), 0o600)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "execution debug", wantProviderCalls: 1, wantPreparationFile: true,
|
|
||||||
createCollision: func(root string, resolved report.Resolved) error {
|
|
||||||
return os.MkdirAll(filepath.Join(workflowDebugRunPath(root, resolved), "execution.json"), 0o700)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg := workflowConfig(t)
|
|
||||||
cfg.Notify.Distributor.Enabled = false
|
|
||||||
now := workflowTime("2026-05-29T08:30:00-05:00")
|
|
||||||
request := GenerateRequest{Config: cfg, Report: ReportDaily, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: now, WorkingDir: t.TempDir()}
|
|
||||||
resolved, err := ResolveGenerate(request, now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
debugRoot := filepath.Join(t.TempDir(), "prompt-debug")
|
|
||||||
if err := test.createCollision(debugRoot, resolved); err != nil {
|
|
||||||
t.Fatalf("create debug collision: %v", err)
|
|
||||||
}
|
|
||||||
bundle := workflowBundle(t)
|
|
||||||
executor := &workflowExecutor{definition: resolved.Definition, raw: []byte(validDailyWorkflowJSON())}
|
|
||||||
request.Collector = &workflowCollector{result: &collect.Result{Bundle: &bundle}}
|
|
||||||
request.Executor = executor
|
|
||||||
request.LLMDebugDir = debugRoot
|
|
||||||
result, err := GenerateDetailed(context.Background(), request)
|
|
||||||
if err == nil || result == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
|
|
||||||
t.Fatalf("result/error/category = %#v/%v/%q", result, err, promptexec.CategoryOf(err))
|
|
||||||
}
|
|
||||||
if executor.providerCalls != test.wantProviderCalls || result.PreparationPath == "" || result.ExecutionPath != "" || result.GeneratedTextRawPath != "" {
|
|
||||||
t.Fatalf("provider/preparation/metadata/execution/raw = %d/%q/%q/%q/%q", executor.providerCalls, result.PreparationPath, result.MetadataPath, result.ExecutionPath, result.GeneratedTextRawPath)
|
|
||||||
}
|
|
||||||
if test.wantPreparationFile && result.MetadataPath == "" {
|
|
||||||
t.Fatal("execution debug failure lost previously persisted metadata")
|
|
||||||
}
|
|
||||||
preparationDebug := filepath.Join(workflowDebugRunPath(debugRoot, resolved), "preparation.json")
|
|
||||||
_, statErr := os.Stat(preparationDebug)
|
|
||||||
if (statErr == nil) != test.wantPreparationFile {
|
|
||||||
t.Fatalf("preparation debug stat error = %v, want file %t", statErr, test.wantPreparationFile)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func workflowDebugRunPath(root string, resolved report.Resolved) string {
|
|
||||||
return filepath.Join(root, string(resolved.Definition.ID), resolved.ValidPeriod.Start.Format("2006-01-02"), resolved.Metadata().RunID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func workflowBundlePaths(id report.ID, validDate, runID string) []string {
|
|
||||||
switch id {
|
|
||||||
case report.Daily:
|
|
||||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md"}
|
|
||||||
case report.Today:
|
|
||||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "today/index.md"}
|
|
||||||
case report.Tomorrow:
|
|
||||||
return []string{"daily/" + validDate + "/" + runID + ".md", "daily/" + validDate + "/index.md", "tomorrow/index.md"}
|
|
||||||
case report.Hourly:
|
|
||||||
return []string{"hourly/index.md"}
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func workflowConfig(t *testing.T) config.Config {
|
|
||||||
t.Helper()
|
|
||||||
cfg := config.Defaults()
|
|
||||||
cfg.Workspace.Root = t.TempDir()
|
|
||||||
cfg.WeatherAPI.Timezone = "America/Chicago"
|
|
||||||
cfg.Location.ID = "home"
|
|
||||||
cfg.Location.Name = "Testville"
|
|
||||||
cfg.Location.Region = "MO"
|
|
||||||
cfg.Notify.Distributor.Enabled = true
|
|
||||||
cfg.Notify.Distributor.PipelineIDTemplate = "reports.{report_id}.{artifact_group}"
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func workflowBundle(t *testing.T) weatherdata.Bundle {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read bundle fixture: %v", err)
|
|
||||||
}
|
|
||||||
var bundle weatherdata.Bundle
|
|
||||||
if err := json.Unmarshal(data, &bundle); err != nil {
|
|
||||||
t.Fatalf("decode bundle fixture: %v", err)
|
|
||||||
}
|
|
||||||
future := bundle.Hourly.Periods[0]
|
|
||||||
future.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
|
||||||
future.EndTime = workflowTime("2026-05-30T07:00:00-05:00")
|
|
||||||
bundle.Hourly.Periods = append(bundle.Hourly.Periods, future)
|
|
||||||
futureNarrative := bundle.Narrative.Periods[0]
|
|
||||||
futureNarrative.StartTime = workflowTime("2026-05-30T06:00:00-05:00")
|
|
||||||
futureNarrative.EndTime = workflowTime("2026-05-30T18:00:00-05:00")
|
|
||||||
futureNarrative.Name = "Tomorrow"
|
|
||||||
bundle.Narrative.Periods = append(bundle.Narrative.Periods, futureNarrative)
|
|
||||||
return bundle
|
|
||||||
}
|
|
||||||
|
|
||||||
func workflowTime(value string) time.Time {
|
|
||||||
parsed, err := time.Parse(time.RFC3339, value)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return parsed
|
|
||||||
}
|
|
||||||
|
|
||||||
func validHourlyWorkflowJSON() string {
|
|
||||||
return `{"summary":"Storm chances increase through late morning.","forecast_discussion":"A front will keep the region unsettled.","precipitation_timing":"A cold front is moving into the region."}`
|
|
||||||
}
|
|
||||||
|
|
||||||
func validTomorrowWorkflowJSON() string {
|
|
||||||
return `{"summary":"Tomorrow starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
|
||||||
}
|
|
||||||
|
|
||||||
func validTodayWorkflowJSON() string {
|
|
||||||
return `{"summary":"Today starts with showers before improving.","forecast_discussion":["Morning showers should taper as drier air arrives.","Afternoon conditions trend quieter."],"precipitation_timing":"The best rain chance is during the morning."}`
|
|
||||||
}
|
|
||||||
|
|
||||||
func validDailyWorkflowJSON() string {
|
|
||||||
return `{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast.","Temperatures stay seasonable by afternoon."],"precipitation_timing":"Rain is most likely during the afternoon."}`
|
|
||||||
}
|
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -25,16 +26,15 @@ type generateSummary struct {
|
|||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
GeneratedAt time.Time `json:"generatedAt"`
|
GeneratedAt time.Time `json:"generatedAt"`
|
||||||
ValidPeriod timeutil.Period `json:"validPeriod"`
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
||||||
ReportPath string `json:"reportPath,omitempty"`
|
|
||||||
OutputPath string `json:"outputPath,omitempty"`
|
OutputPath string `json:"outputPath,omitempty"`
|
||||||
MetadataPath string `json:"metadataPath,omitempty"`
|
|
||||||
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
|
||||||
PreparationPath string `json:"preparationPath,omitempty"`
|
|
||||||
ExecutionPath string `json:"executionPath,omitempty"`
|
|
||||||
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
||||||
GeneratedTextRawPath string `json:"generatedTextRawPath,omitempty"`
|
PromptVersion string `json:"promptVersion"`
|
||||||
GeneratedTextPath string `json:"generatedTextPath,omitempty"`
|
Timezone string `json:"timezone"`
|
||||||
RenderContextPath string `json:"renderContextPath,omitempty"`
|
ProfileID string `json:"profileId,omitempty"`
|
||||||
|
BackendID string `json:"backendId,omitempty"`
|
||||||
|
ModelName string `json:"modelName,omitempty"`
|
||||||
|
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
|
||||||
|
ValidationStatus string `json:"validationStatus,omitempty"`
|
||||||
Notification *generateNotificationSummary `json:"notification,omitempty"`
|
Notification *generateNotificationSummary `json:"notification,omitempty"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -73,24 +73,20 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
|||||||
return summary
|
return summary
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata := result.Metadata
|
summary.ReportID = result.ReportID
|
||||||
summary.ReportID = metadata.ReportID
|
summary.ReportName = result.ReportName
|
||||||
summary.ReportName = reportName(metadata.ReportID)
|
summary.PromptID = result.PromptID
|
||||||
summary.PromptID = metadata.PromptID
|
summary.PromptVersion = result.PromptVersion
|
||||||
summary.RunID = metadata.RunID
|
summary.RunID = result.RunID
|
||||||
summary.Status = summaryStatusSucceeded
|
summary.Status = summaryStatusSucceeded
|
||||||
summary.GeneratedAt = metadata.GeneratedAt
|
summary.GeneratedAt = result.GeneratedAt
|
||||||
summary.ValidPeriod = metadata.ValidPeriod
|
summary.ValidPeriod = result.ValidPeriod
|
||||||
summary.ReportPath = result.ReportPath
|
summary.Timezone = result.Timezone
|
||||||
|
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
|
||||||
|
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
|
||||||
|
summary.ValidationStatus = string(result.ValidationStatus)
|
||||||
summary.OutputPath = result.OutputPath
|
summary.OutputPath = result.OutputPath
|
||||||
summary.MetadataPath = result.MetadataPath
|
|
||||||
summary.DataPackagePath = result.DataPackagePath
|
|
||||||
summary.PreparationPath = result.PreparationPath
|
|
||||||
summary.ExecutionPath = result.ExecutionPath
|
|
||||||
summary.LLMDebugPath = result.LLMDebugPath
|
summary.LLMDebugPath = result.LLMDebugPath
|
||||||
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
|
|
||||||
summary.GeneratedTextPath = result.GeneratedTextPath
|
|
||||||
summary.RenderContextPath = result.RenderContextPath
|
|
||||||
summary.Notification = newGenerateNotificationSummary(result.Notification)
|
summary.Notification = newGenerateNotificationSummary(result.Notification)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
summary.Status = summaryStatusFailed
|
summary.Status = summaryStatusFailed
|
||||||
|
|||||||
@@ -2,264 +2,29 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewGenerateSummaryForGeneratedTextReport(t *testing.T) {
|
func TestGenerateSummaryUsesActiveResultFields(t *testing.T) {
|
||||||
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
||||||
acceptedAt := generatedAt.Add(time.Minute)
|
summary := newGenerateSummary(&app.ReportResult{ReportID: report.Daily, ReportName: "Daily Report", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", RunID: "run-123", GeneratedAt: generatedAt, Timezone: "America/Chicago", ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "model", ValidationStatus: promptexec.ValidationPassed, OutputPath: "/reports/daily.md"}, nil)
|
||||||
startedAt := acceptedAt.Add(time.Minute)
|
if summary.OutputPath == "" || summary.ProfileID == "" || summary.ValidationStatus != string(promptexec.ValidationPassed) {
|
||||||
finishedAt := startedAt.Add(time.Minute)
|
t.Fatalf("summary = %#v", summary)
|
||||||
result := &app.ReportResult{
|
|
||||||
DataPackagePath: "/runs/hourly/data_package.yaml",
|
|
||||||
PreparationPath: "/runs/hourly/preparation.json",
|
|
||||||
ExecutionPath: "/runs/hourly/execution.json",
|
|
||||||
LLMDebugPath: "/operator-debug/hourly/2026-05-29/run-123",
|
|
||||||
ReportPath: "/runs/hourly/report.md",
|
|
||||||
OutputPath: "/copies/hourly.md",
|
|
||||||
MetadataPath: "/runs/hourly/metadata.json",
|
|
||||||
GeneratedTextRawPath: "/runs/hourly/generated_text_raw.json",
|
|
||||||
GeneratedTextPath: "/runs/hourly/generated_text.json",
|
|
||||||
RenderContextPath: "/runs/hourly/render_context.json",
|
|
||||||
Metadata: state.Metadata{
|
|
||||||
ReportID: report.Hourly,
|
|
||||||
PromptID: "weather.hourly_generated_text",
|
|
||||||
RunID: "20260529T133000Z_hourly",
|
|
||||||
GeneratedAt: generatedAt,
|
|
||||||
ValidPeriod: testSummaryPeriod(generatedAt),
|
|
||||||
},
|
|
||||||
Notification: &app.NotificationResult{
|
|
||||||
Status: "succeeded",
|
|
||||||
UploadStatus: "accepted",
|
|
||||||
RunID: "distributor-run",
|
|
||||||
PipelineID: "weatherreporter.hourly",
|
|
||||||
BundleID: "weatherreporter.home.hourly",
|
|
||||||
IdempotencyKey: "weatherreporter.home.hourly.20260529T133000Z_hourly",
|
|
||||||
AcceptedAt: acceptedAt,
|
|
||||||
StartedAt: &startedAt,
|
|
||||||
FinishedAt: &finishedAt,
|
|
||||||
Report: []byte(`{"actions":[{"action":"replace_older"}]}`),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := newGenerateSummary(result, nil)
|
|
||||||
|
|
||||||
if summary.Command != "generate" || summary.Status != "succeeded" {
|
|
||||||
t.Fatalf("summary command/status = %q/%q, want generate/succeeded", summary.Command, summary.Status)
|
|
||||||
}
|
|
||||||
if summary.ReportID != report.Hourly || summary.ReportName != "Hourly Report" || summary.PromptID != "weather.hourly_generated_text" || summary.RunID != "20260529T133000Z_hourly" {
|
|
||||||
t.Fatalf("summary identity = %#v, want hourly report identity", summary)
|
|
||||||
}
|
|
||||||
if summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.LLMDebugPath == "" || summary.GeneratedTextRawPath == "" || summary.GeneratedTextPath == "" || summary.RenderContextPath == "" {
|
|
||||||
t.Fatalf("generated-text paths = %#v, want generated-text artifact paths", summary)
|
|
||||||
}
|
|
||||||
if summary.Notification == nil || summary.Notification.RunID != "distributor-run" || summary.Notification.AcceptedAt == nil || !summary.Notification.AcceptedAt.Equal(acceptedAt) {
|
|
||||||
t.Fatalf("notification = %#v, want summarized distributor result", summary.Notification)
|
|
||||||
}
|
}
|
||||||
data, err := json.Marshal(summary)
|
data, err := json.Marshal(summary)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
}
|
}
|
||||||
if strings.Contains(string(data), "replace_older") || strings.Contains(string(data), "actions") {
|
for _, forbidden := range []string{"reportPath", "metadataPath", "dataPackagePath", "preparationPath", "executionPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath"} {
|
||||||
t.Fatalf("summary JSON includes raw distributor report payload:\n%s", string(data))
|
if strings.Contains(string(data), forbidden) {
|
||||||
}
|
t.Fatalf("summary includes %q: %s", forbidden, data)
|
||||||
if strings.Contains(string(data), "preflightPath") || strings.Contains(string(data), "generatedTextResultPath") || !strings.Contains(string(data), "preparationPath") || !strings.Contains(string(data), "executionPath") {
|
|
||||||
t.Fatalf("summary JSON does not use prompt artifact path names:\n%s", string(data))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewGenerateSummaryOmitsNotificationWhenNotAttempted(t *testing.T) {
|
|
||||||
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
||||||
result := &app.ReportResult{
|
|
||||||
DataPackagePath: "/runs/daily/data_package.yaml",
|
|
||||||
PreparationPath: "/runs/daily/preparation.json",
|
|
||||||
ReportPath: "/runs/daily/report.md",
|
|
||||||
OutputPath: "/copies/daily.md",
|
|
||||||
MetadataPath: "/runs/daily/metadata.json",
|
|
||||||
Metadata: state.Metadata{
|
|
||||||
ReportID: report.Daily,
|
|
||||||
PromptID: "weather.daily_generated_text",
|
|
||||||
RunID: "20260529T133000Z_daily",
|
|
||||||
GeneratedAt: generatedAt,
|
|
||||||
ValidPeriod: testSummaryPeriod(generatedAt),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
summary := newGenerateSummary(result, nil)
|
|
||||||
|
|
||||||
if summary.ReportID != report.Daily || summary.ReportName != "Daily Report" || summary.Status != "succeeded" {
|
|
||||||
t.Fatalf("summary = %#v, want successful daily summary", summary)
|
|
||||||
}
|
|
||||||
if summary.Notification != nil {
|
|
||||||
t.Fatalf("notification summary = %#v, want omitted", summary.Notification)
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(summary)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
for _, omitted := range []string{"notification"} {
|
|
||||||
if strings.Contains(string(data), omitted) {
|
|
||||||
t.Fatalf("summary JSON contains %q, want omitted:\n%s", omitted, string(data))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewGenerateSummaryOmitsUnreachedArtifactPaths(t *testing.T) {
|
|
||||||
result := &app.ReportResult{
|
|
||||||
DataPackagePath: "/runs/daily/data_package.yaml",
|
|
||||||
PreparationPath: "/runs/daily/preparation.json",
|
|
||||||
Metadata: state.Metadata{
|
|
||||||
ReportID: report.Daily,
|
|
||||||
RunID: "20260529T133000Z_daily",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.Marshal(newGenerateSummary(result, errors.New("metadata write failed")))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
text := string(data)
|
|
||||||
for _, omitted := range []string{"executionPath", "reportPath", "outputPath", "metadataPath", "generatedTextRawPath", "generatedTextPath", "renderContextPath", "notificationPath"} {
|
|
||||||
if strings.Contains(text, omitted) {
|
|
||||||
t.Fatalf("partial summary includes unreached field %q:\n%s", omitted, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(text, "dataPackagePath") || !strings.Contains(text, "preparationPath") {
|
|
||||||
t.Fatalf("partial summary omits reached paths:\n%s", text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewGenerateSummaryForNotificationFailure(t *testing.T) {
|
|
||||||
generatedAt := time.Date(2026, 5, 29, 13, 30, 0, 0, time.UTC)
|
|
||||||
result := &app.ReportResult{
|
|
||||||
DataPackagePath: "/runs/hourly/data_package.yaml",
|
|
||||||
PreparationPath: "/runs/hourly/preparation.json",
|
|
||||||
ReportPath: "/runs/hourly/report.md",
|
|
||||||
OutputPath: "/copies/hourly.md",
|
|
||||||
MetadataPath: "/runs/hourly/metadata.json",
|
|
||||||
Metadata: state.Metadata{
|
|
||||||
ReportID: report.Hourly,
|
|
||||||
PromptID: "weather.hourly_generated_text",
|
|
||||||
RunID: "20260529T133000Z_hourly",
|
|
||||||
GeneratedAt: generatedAt,
|
|
||||||
ValidPeriod: testSummaryPeriod(generatedAt),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
err := errors.New(`notify report "hourly" run "20260529T133000Z_hourly": upload rejected`)
|
|
||||||
|
|
||||||
summary := newGenerateSummary(result, err)
|
|
||||||
|
|
||||||
if summary.Status != "failed" || summary.Error != err.Error() {
|
|
||||||
t.Fatalf("status/error = %q/%q, want failed notification error", summary.Status, summary.Error)
|
|
||||||
}
|
|
||||||
if summary.ReportPath == "" || summary.MetadataPath == "" {
|
|
||||||
t.Fatalf("artifact paths = report %q metadata %q, want retained output provenance", summary.ReportPath, summary.MetadataPath)
|
|
||||||
}
|
|
||||||
data, marshalErr := json.Marshal(summary)
|
|
||||||
if marshalErr != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", marshalErr)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(data), "notificationPath") {
|
|
||||||
t.Fatalf("summary JSON includes a notification receipt path:\n%s", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewBatchSummaryStatusDerivation(t *testing.T) {
|
|
||||||
startedAt := time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)
|
|
||||||
finishedAt := startedAt.Add(2 * time.Minute)
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
result *app.BatchResult
|
|
||||||
wantStatus string
|
|
||||||
wantError string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "success",
|
|
||||||
result: &app.BatchResult{
|
|
||||||
Batch: app.BatchMorning,
|
|
||||||
StartedAt: startedAt,
|
|
||||||
FinishedAt: finishedAt,
|
|
||||||
Total: 1,
|
|
||||||
Succeeded: 1,
|
|
||||||
Reports: []app.BatchReportResult{{ReportID: report.Today, Status: "succeeded"}},
|
|
||||||
},
|
|
||||||
wantStatus: "succeeded",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "report failure",
|
|
||||||
result: &app.BatchResult{
|
|
||||||
Batch: app.BatchMorning,
|
|
||||||
Total: 2,
|
|
||||||
Succeeded: 1,
|
|
||||||
Failed: 1,
|
|
||||||
Reports: []app.BatchReportResult{
|
|
||||||
{ReportID: report.Today, Status: "succeeded"},
|
|
||||||
{ReportID: report.Tomorrow, Status: "failed", Error: "render failed"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
wantStatus: "failed",
|
|
||||||
wantError: "batch morning failed: 1 of 2 reports failed",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "skipped notification",
|
|
||||||
result: &app.BatchResult{
|
|
||||||
Batch: app.BatchEvening,
|
|
||||||
Total: 2,
|
|
||||||
Succeeded: 1,
|
|
||||||
Failed: 1,
|
|
||||||
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "failed"}},
|
|
||||||
Notification: &app.BatchNotificationResult{
|
|
||||||
Status: "skipped",
|
|
||||||
Reason: "one or more reports failed",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
wantStatus: "failed",
|
|
||||||
wantError: "batch evening failed: 1 of 2 reports failed",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "failed notification",
|
|
||||||
result: &app.BatchResult{
|
|
||||||
Batch: app.BatchEvening,
|
|
||||||
Total: 1,
|
|
||||||
Succeeded: 1,
|
|
||||||
Reports: []app.BatchReportResult{{ReportID: report.Tomorrow, Status: "succeeded"}},
|
|
||||||
Notification: &app.BatchNotificationResult{
|
|
||||||
Status: "failed",
|
|
||||||
Error: "notify batch evening: upload rejected",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
wantStatus: "failed",
|
|
||||||
wantError: "batch evening notification failed: notify batch evening: upload rejected",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
summary := newBatchSummary(tt.result)
|
|
||||||
if summary.Command != "run" || summary.Status != tt.wantStatus {
|
|
||||||
t.Fatalf("command/status = %q/%q, want run/%s", summary.Command, summary.Status, tt.wantStatus)
|
|
||||||
}
|
|
||||||
if summary.Error != tt.wantError {
|
|
||||||
t.Fatalf("error = %q, want %q", summary.Error, tt.wantError)
|
|
||||||
}
|
|
||||||
if len(summary.Reports) != len(tt.result.Reports) {
|
|
||||||
t.Fatalf("reports = %#v, want copied report list", summary.Reports)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testSummaryPeriod(start time.Time) timeutil.Period {
|
|
||||||
return timeutil.Period{
|
|
||||||
Start: start,
|
|
||||||
End: start.Add(6 * time.Hour),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,631 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testRenderedPrompt = "PRIVATE RENDERED PROMPT"
|
|
||||||
testSchemaBody = `{"private":"schema"}`
|
|
||||||
testDataBody = "PRIVATE DATA PACKAGE"
|
|
||||||
testGeneratedBody = "PRIVATE GENERATED BODY"
|
|
||||||
testEndpoint = "https://user:credential@example.invalid/v1?token=credential"
|
|
||||||
testParameters = `{"temperature":0.2,"private":"parameter"}`
|
|
||||||
testCredential = "cli-secret-credential"
|
|
||||||
)
|
|
||||||
|
|
||||||
type commandOutput struct {
|
|
||||||
stdout string
|
|
||||||
stderr string
|
|
||||||
}
|
|
||||||
|
|
||||||
type cliExecutor struct {
|
|
||||||
fail bool
|
|
||||||
failPrompt string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e cliExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
|
||||||
name := strings.TrimSuffix(strings.TrimPrefix(id, "weather."), "_generated_text")
|
|
||||||
return promptexec.PromptInspection{
|
|
||||||
PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "offline-profile",
|
|
||||||
Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
|
|
||||||
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: name + ".generated_text.schema.json"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e cliExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
||||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "offline", ModelName: "offline-model"}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e cliExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
||||||
now := time.Date(2026, 5, 29, 12, 1, 0, 0, time.UTC)
|
|
||||||
preparation := promptexec.Preparation{
|
|
||||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
|
||||||
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", DataPackagePath: req.DataPackagePath,
|
|
||||||
StartedAt: now, EndedAt: now,
|
|
||||||
}
|
|
||||||
if err := callback(preparation, nil); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if e.fail || req.PromptID == e.failPrompt {
|
|
||||||
return nil, errors.New(strings.Join([]string{
|
|
||||||
"provider failed", testEndpoint, testCredential, testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testParameters,
|
|
||||||
}, " "))
|
|
||||||
}
|
|
||||||
raw := []byte(`{"summary":"Showers are possible.","forecast_discussion":["Rain chances continue."],"precipitation_timing":"Rain is most likely this afternoon."}`)
|
|
||||||
if req.PromptID == "weather.hourly_generated_text" {
|
|
||||||
raw = []byte(`{"summary":"Storm chances increase.","forecast_discussion":"A front keeps the area unsettled.","precipitation_timing":"Rain is most likely late this morning."}`)
|
|
||||||
}
|
|
||||||
return &promptexec.Execution{
|
|
||||||
RunID: "offline-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
|
||||||
ProfileID: req.ProfileID, BackendID: "offline", ModelName: "offline-model", GeneratedHash: "generated-hash",
|
|
||||||
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath, RawOutput: raw,
|
|
||||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "generated_text.schema.json", nil),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerHelpListsOnlySupportedCommands(t *testing.T) {
|
|
||||||
output, err := runCLICommand(Runner{}, "--help")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run(--help) error = %v", err)
|
|
||||||
}
|
|
||||||
for _, command := range []string{
|
|
||||||
"--version",
|
|
||||||
"generate daily", "generate today", "generate tomorrow", "generate hourly", "run morning", "run evening",
|
|
||||||
"inspect reports", "inspect metadata", "inspect modules", "inspect data-package", "inspect prior", "inspect sources",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(output.stdout, command) {
|
|
||||||
t.Fatalf("help missing %q:\n%s", command, output.stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, retired := range []string{"near-term", "three-day", "weekend", "storm"} {
|
|
||||||
if strings.Contains(output.stdout, retired) {
|
|
||||||
t.Fatalf("help contains retired command %q:\n%s", retired, output.stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, description := range []string{"Write the generated Markdown report to PATH.", "Write generated Markdown reports beneath PATH"} {
|
|
||||||
if !strings.Contains(output.stdout, description) {
|
|
||||||
t.Fatalf("help missing output description %q:\n%s", description, output.stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerVersion(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
runner Runner
|
|
||||||
version string
|
|
||||||
}{
|
|
||||||
{name: "development default", runner: Runner{}, version: "development"},
|
|
||||||
{name: "injected release", runner: Runner{Version: "v0.9.0-test"}, version: "v0.9.0-test"},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
output, err := runCLICommand(test.runner, "--version")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run(--version) error = %v", err)
|
|
||||||
}
|
|
||||||
if output.stdout != "weatherreporter "+test.version+"\n" || output.stderr != "" {
|
|
||||||
t.Fatalf("Run(--version) output = stdout %q stderr %q", output.stdout, output.stderr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if _, err := runCLICommand(Runner{Version: "v0.9.0-test"}, "--version", "extra"); err == nil {
|
|
||||||
t.Fatal("Run(--version extra) error = nil")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveSupportedCommandsAndFlags(t *testing.T) {
|
|
||||||
configPath := writeCLIConfig(t, t.TempDir(), "")
|
|
||||||
runner, constructions := countingRunner(cliExecutor{})
|
|
||||||
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
args []string
|
|
||||||
want app.ReportKind
|
|
||||||
}{
|
|
||||||
{name: "daily", args: []string{"daily", "--date", "2026-05-29"}, want: app.ReportDaily},
|
|
||||||
{name: "today", args: []string{"today"}, want: app.ReportToday},
|
|
||||||
{name: "tomorrow", args: []string{"tomorrow"}, want: app.ReportTomorrow},
|
|
||||||
{name: "hourly", args: []string{"hourly"}, want: app.ReportHourly},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
args := append(tt.args, "--config", configPath)
|
|
||||||
req, err := runner.resolveGenerate(args)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveGenerate() error = %v", err)
|
|
||||||
}
|
|
||||||
if req.Report != tt.want || req.Executor == nil {
|
|
||||||
t.Fatalf("request = %#v, want report %q with executor", req, tt.want)
|
|
||||||
}
|
|
||||||
if tt.want == app.ReportDaily || tt.want == app.ReportToday {
|
|
||||||
if got := req.Date.Format(timeutil.DateLayout); got != "2026-05-29" {
|
|
||||||
t.Fatalf("resolved date = %q, want 2026-05-29", got)
|
|
||||||
}
|
|
||||||
} else if !req.Date.IsZero() {
|
|
||||||
t.Fatalf("resolved date = %s, want unset", req.Date)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
want app.BatchKind
|
|
||||||
}{
|
|
||||||
{name: "morning", want: app.BatchMorning},
|
|
||||||
{name: "evening", want: app.BatchEvening},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
req, err := runner.resolveRun([]string{tt.name, "--config", configPath})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveRun() error = %v", err)
|
|
||||||
}
|
|
||||||
if req.Batch != tt.want || req.Executor == nil {
|
|
||||||
t.Fatalf("request = %#v, want batch %q with executor", req, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if *constructions != 6 {
|
|
||||||
t.Fatalf("executor constructions = %d, want one per resolved action", *constructions)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveGenerateAndRunApplySharedActionFlags(t *testing.T) {
|
|
||||||
configPath := writeCLIConfig(t, t.TempDir(), "")
|
|
||||||
runner, _ := countingRunner(cliExecutor{})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
|
|
||||||
generate, generateOpts, err := runner.resolveGenerateAction([]string{
|
|
||||||
"daily", "--config", configPath, "--date", "2026-05-30", "--units", "metric", "--tz", "UTC",
|
|
||||||
"--out", "daily.md", "--llm-debug-dir", "/safe/debug", "--quiet",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveGenerateAction() error = %v", err)
|
|
||||||
}
|
|
||||||
if generate.Config.WeatherAPI.Units != "metric" || generate.Config.WeatherAPI.Timezone != "UTC" || generate.OutputPath != filepath.Join(runner.WorkingDir, "daily.md") || generate.WorkingDir != runner.WorkingDir || generate.LLMDebugDir != "/safe/debug" || !generateOpts.Quiet {
|
|
||||||
t.Fatalf("generate request/options = %#v/%#v", generate, generateOpts)
|
|
||||||
}
|
|
||||||
if got := generate.Date.Format(timeutil.DateLayout); got != "2026-05-30" {
|
|
||||||
t.Fatalf("generate date = %q, want 2026-05-30", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
batch, batchOpts, err := runner.resolveRunAction([]string{
|
|
||||||
"evening", "--config", configPath, "--units", "metric", "--tz", "UTC", "--out-dir", "reports",
|
|
||||||
"--llm-debug-dir", "/safe/debug", "--quiet",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolveRunAction() error = %v", err)
|
|
||||||
}
|
|
||||||
if batch.Config.WeatherAPI.Units != "metric" || batch.Config.WeatherAPI.Timezone != "UTC" || batch.OutputDir != filepath.Join(runner.WorkingDir, "reports") || batch.WorkingDir != runner.WorkingDir || batch.LLMDebugDir != "/safe/debug" || !batchOpts.Quiet {
|
|
||||||
t.Fatalf("batch request/options = %#v/%#v", batch, batchOpts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateFlagContracts(t *testing.T) {
|
|
||||||
for _, kind := range []app.ReportKind{app.ReportDaily, app.ReportToday, app.ReportTomorrow, app.ReportHourly} {
|
|
||||||
t.Run(string(kind), func(t *testing.T) {
|
|
||||||
opts, err := parseGenerateFlags(kind, []string{"--llm-debug-dir", "/safe/debug", "--quiet", "--out", "report.md"})
|
|
||||||
if err != nil || opts.LLMDebugDir != "/safe/debug" || !opts.Quiet || opts.Output != "report.md" {
|
|
||||||
t.Fatalf("parseGenerateFlags() = %#v, %v", opts, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
runner, _ := countingRunner(cliExecutor{})
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
args []string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "daily requires date", args: []string{"daily"}, want: "requires --date"},
|
|
||||||
{name: "malformed daily date", args: []string{"daily", "--date", "bad-date"}, want: "YYYY-MM-DD"},
|
|
||||||
{name: "malformed today date", args: []string{"today", "--date", "bad-date"}, want: "YYYY-MM-DD"},
|
|
||||||
{name: "tomorrow rejects date", args: []string{"tomorrow", "--date", "2026-05-29"}},
|
|
||||||
{name: "hourly rejects date", args: []string{"hourly", "--date", "2026-05-29"}},
|
|
||||||
{name: "hourly rejects hours", args: []string{"hourly", "--hours", "6"}},
|
|
||||||
{name: "hourly rejects duration", args: []string{"hourly", "--duration", "6h"}},
|
|
||||||
{name: "batch rejects output", args: []string{"run", "--out", "report.md"}},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
var err error
|
|
||||||
if tt.args[0] == "run" {
|
|
||||||
_, err = runner.resolveRun(append([]string{"morning"}, tt.args[1:]...))
|
|
||||||
} else {
|
|
||||||
_, err = runner.resolveGenerate(tt.args)
|
|
||||||
}
|
|
||||||
if err == nil || (tt.want != "" && !strings.Contains(err.Error(), tt.want)) {
|
|
||||||
t.Fatalf("error = %v, want rejection containing %q", err, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolversRejectRetiredAndUnknownNames(t *testing.T) {
|
|
||||||
runner, _ := countingRunner(cliExecutor{})
|
|
||||||
for _, name := range []string{"near-term", "three-day", "weekend", "storm"} {
|
|
||||||
if _, err := runner.resolveGenerate([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown generate report") {
|
|
||||||
t.Fatalf("resolveGenerate(%q) error = %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, name := range []string{"daily", "weekend", "storm"} {
|
|
||||||
if _, err := runner.resolveRun([]string{name}); err == nil || !strings.Contains(err.Error(), "unknown run batch") {
|
|
||||||
t.Fatalf("resolveRun(%q) error = %v", name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerSuccessfulSingleAndBatchActions(t *testing.T) {
|
|
||||||
for _, tt := range []struct {
|
|
||||||
name string
|
|
||||||
args func(string, string) []string
|
|
||||||
}{
|
|
||||||
{name: "single", args: func(configPath, outputPath string) []string {
|
|
||||||
return []string{"generate", "today", "--config", configPath, "--out", outputPath}
|
|
||||||
}},
|
|
||||||
{name: "batch", args: func(configPath, outputPath string) []string {
|
|
||||||
return []string{"run", "evening", "--config", configPath, "--out-dir", outputPath}
|
|
||||||
}},
|
|
||||||
} {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runner, constructions := countingRunner(cliExecutor{})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
output, err := runCLICommand(runner, tt.args(fixture.configPath, fixture.path("copies"))...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run() error = %v", err)
|
|
||||||
}
|
|
||||||
if *constructions != 1 {
|
|
||||||
t.Fatalf("executor constructions = %d, want 1", *constructions)
|
|
||||||
}
|
|
||||||
assertRoutineOutputSafe(t, output)
|
|
||||||
if tt.name == "single" {
|
|
||||||
summary := decodeGenerateSummary(t, output.stdout)
|
|
||||||
if summary.Status != summaryStatusSucceeded || summary.ReportID != report.Today || summary.ReportPath == "" || summary.OutputPath == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" {
|
|
||||||
t.Fatalf("single summary = %#v", summary)
|
|
||||||
}
|
|
||||||
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
|
|
||||||
t.Fatalf("single summary contains absent optional fields:\n%s", output.stdout)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
summary := decodeBatchSummary(t, output.stdout)
|
|
||||||
if summary.Status != summaryStatusSucceeded || summary.Batch != app.BatchEvening || summary.Total != 1 || len(summary.Reports) != 1 || summary.Reports[0].OutputPath == "" {
|
|
||||||
t.Fatalf("batch summary = %#v", summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(output.stderr, "report=tomorrow status=succeeded") || !strings.Contains(output.stderr, "batch=evening total=1 succeeded=1 failed=0") {
|
|
||||||
t.Fatalf("batch status = %q", output.stderr)
|
|
||||||
}
|
|
||||||
if strings.Contains(output.stdout, `"notification"`) || strings.Contains(output.stdout, `"llmDebugPath"`) {
|
|
||||||
t.Fatalf("batch summary contains absent optional fields:\n%s", output.stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPreRunFailureAndQuietMode(t *testing.T) {
|
|
||||||
runner, constructions := countingRunner(cliExecutor{})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
output, err := runCLICommand(runner, "generate", "daily")
|
|
||||||
if err == nil || output.stdout != "" || output.stderr != "" {
|
|
||||||
t.Fatalf("pre-run output/error = %#v/%v, want error without summary", output, err)
|
|
||||||
}
|
|
||||||
if *constructions != 1 {
|
|
||||||
t.Fatalf("executor constructions = %d, want one action-scoped construction", *constructions)
|
|
||||||
}
|
|
||||||
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runner, _ = countingRunner(cliExecutor{})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
output, err = runCLICommand(runner, "generate", "today", "--config", fixture.configPath, "--quiet")
|
|
||||||
if err != nil || output.stdout != "" || output.stderr != "" {
|
|
||||||
t.Fatalf("quiet output/error = %#v/%v", output, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerFailedActionReportsSafePartialSummary(t *testing.T) {
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runner, constructions := countingRunner(cliExecutor{fail: true})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
output, err := runCLICommand(runner, "generate", "today", "--config", fixture.configPath)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Run() error = nil, want execution failure")
|
|
||||||
}
|
|
||||||
if *constructions != 1 {
|
|
||||||
t.Fatalf("executor constructions = %d, want 1", *constructions)
|
|
||||||
}
|
|
||||||
summary := decodeGenerateSummary(t, output.stdout)
|
|
||||||
if summary.Status != summaryStatusFailed || summary.RunID == "" || summary.MetadataPath == "" || summary.DataPackagePath == "" || summary.PreparationPath == "" || summary.ExecutionPath == "" || summary.ReportPath != "" {
|
|
||||||
t.Fatalf("failed summary paths = %#v", summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(summary.Error, "prompt execution failed") {
|
|
||||||
t.Fatalf("failed summary error = %q", summary.Error)
|
|
||||||
}
|
|
||||||
assertRoutineOutputSafe(t, output)
|
|
||||||
|
|
||||||
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
|
|
||||||
inspected, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, summary.RunID)
|
|
||||||
if inspectErr != nil {
|
|
||||||
t.Fatalf("inspect %s error = %v", command, inspectErr)
|
|
||||||
}
|
|
||||||
if !strings.Contains(inspected.stdout, summary.RunID) {
|
|
||||||
t.Fatalf("inspect %s missing failed run id:\n%s", command, inspected.stdout)
|
|
||||||
}
|
|
||||||
assertRoutineOutputSafe(t, inspected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerMixedBatchReportsSafePartialFailure(t *testing.T) {
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runner, constructions := countingRunner(cliExecutor{failPrompt: "weather.tomorrow_generated_text"})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
output, err := runCLICommand(runner, "run", "morning", "--config", fixture.configPath)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Run() error = nil, want aggregate batch failure")
|
|
||||||
}
|
|
||||||
if *constructions != 1 {
|
|
||||||
t.Fatalf("executor constructions = %d, want 1", *constructions)
|
|
||||||
}
|
|
||||||
summary := decodeBatchSummary(t, output.stdout)
|
|
||||||
if summary.Status != summaryStatusFailed || summary.Total != 2 || summary.Succeeded != 1 || summary.Failed != 1 || len(summary.Reports) != 2 {
|
|
||||||
t.Fatalf("failed batch summary = %#v", summary)
|
|
||||||
}
|
|
||||||
var succeeded, failed *app.BatchReportResult
|
|
||||||
for index := range summary.Reports {
|
|
||||||
item := &summary.Reports[index]
|
|
||||||
if item.Status == summaryStatusSucceeded {
|
|
||||||
succeeded = item
|
|
||||||
} else if item.Status == summaryStatusFailed {
|
|
||||||
failed = item
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if succeeded == nil || succeeded.ReportPath == "" || succeeded.MetadataPath == "" || succeeded.ExecutionPath == "" {
|
|
||||||
t.Fatalf("successful batch item paths = %#v", succeeded)
|
|
||||||
}
|
|
||||||
if failed == nil || failed.ReportPath != "" || failed.MetadataPath == "" || failed.DataPackagePath == "" || failed.PreparationPath == "" || failed.ExecutionPath == "" {
|
|
||||||
t.Fatalf("failed batch item paths = %#v", failed)
|
|
||||||
}
|
|
||||||
if !strings.Contains(output.stderr, "status=succeeded") || !strings.Contains(output.stderr, "status=failed") || !strings.Contains(output.stderr, "batch=morning total=2 succeeded=1 failed=1") {
|
|
||||||
t.Fatalf("partial batch status = %q", output.stderr)
|
|
||||||
}
|
|
||||||
assertRoutineOutputSafe(t, output)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectsReportsAndCurrentArtifacts(t *testing.T) {
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runner, _ := countingRunner(cliExecutor{})
|
|
||||||
runner.WorkingDir = t.TempDir()
|
|
||||||
first := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 11, 0, 0, 0, time.UTC))
|
|
||||||
second := runSuccessfulGenerate(t, runner, fixture.configPath, time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC))
|
|
||||||
|
|
||||||
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath, "--limit", "2")
|
|
||||||
if err != nil || !strings.Contains(listed.stdout, first.RunID) || !strings.Contains(listed.stdout, second.RunID) {
|
|
||||||
t.Fatalf("inspect reports output/error = %s/%v", listed.stdout, err)
|
|
||||||
}
|
|
||||||
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
|
|
||||||
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, second.RunID)
|
|
||||||
if inspectErr != nil || !strings.Contains(output.stdout, second.RunID) {
|
|
||||||
t.Fatalf("inspect %s output/error = %s/%v", command, output.stdout, inspectErr)
|
|
||||||
}
|
|
||||||
assertRoutineOutputSafe(t, output)
|
|
||||||
}
|
|
||||||
prior, err := runCLICommand(runner, "inspect", "prior", "--config", fixture.configPath, second.RunID)
|
|
||||||
if err != nil || !strings.Contains(prior.stdout, first.RunID) {
|
|
||||||
t.Fatalf("inspect prior output/error = %s/%v", prior.stdout, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectsHistoricalMetadataAndArtifacts(t *testing.T) {
|
|
||||||
fixture := newCLIFixture(t)
|
|
||||||
runIDs := []string{
|
|
||||||
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("three_day"), time.Date(2026, 5, 20, 12, 0, 0, 0, time.UTC)),
|
|
||||||
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("weekend"), time.Date(2026, 5, 21, 12, 0, 0, 0, time.UTC)),
|
|
||||||
writeHistoricalInspectionFixture(t, fixture.workspaceRoot, report.ID("storm"), time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC)),
|
|
||||||
}
|
|
||||||
runID := runIDs[0]
|
|
||||||
runner, _ := countingRunner(cliExecutor{})
|
|
||||||
|
|
||||||
listed, err := runCLICommand(runner, "inspect", "reports", "--config", fixture.configPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect historical reports output/error = %s/%v", listed.stdout, err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{runIDs[0], runIDs[1], runIDs[2], `"reportId": "three_day"`, `"reportId": "weekend"`, `"reportId": "storm"`} {
|
|
||||||
if !strings.Contains(listed.stdout, want) {
|
|
||||||
t.Fatalf("inspect historical reports missing %q:\n%s", want, listed.stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, command := range []string{"metadata", "modules", "data-package", "sources"} {
|
|
||||||
output, inspectErr := runCLICommand(runner, "inspect", command, "--config", fixture.configPath, runID)
|
|
||||||
if inspectErr != nil || !strings.Contains(output.stdout, runID) {
|
|
||||||
t.Fatalf("inspect historical %s output/error = %s/%v", command, output.stdout, inspectErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
metadata, err := runCLICommand(runner, "inspect", "metadata", "--config", fixture.configPath, runID)
|
|
||||||
if err != nil || !strings.Contains(metadata.stdout, `"schemaVersion": "weatherreporter.metadata.v1"`) || !strings.Contains(metadata.stdout, `"preflightPath"`) || strings.Contains(metadata.stdout, `"preparationPath"`) {
|
|
||||||
t.Fatalf("historical metadata aliases/output = %s/%v", metadata.stdout, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func countingRunner(executor promptexec.Executor) (Runner, *int) {
|
|
||||||
count := new(int)
|
|
||||||
return Runner{
|
|
||||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC)},
|
|
||||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
|
||||||
*count++
|
|
||||||
return executor, nil
|
|
||||||
},
|
|
||||||
}, count
|
|
||||||
}
|
|
||||||
|
|
||||||
func runCLICommand(runner Runner, args ...string) (commandOutput, error) {
|
|
||||||
var stdout bytes.Buffer
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
err := runner.Run(context.Background(), args, &stdout, &stderr)
|
|
||||||
return commandOutput{stdout: stdout.String(), stderr: stderr.String()}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func runSuccessfulGenerate(t *testing.T, base Runner, configPath string, now time.Time) generateSummary {
|
|
||||||
t.Helper()
|
|
||||||
base.Clock = timeutil.FixedClock{Time: now}
|
|
||||||
output, err := runCLICommand(base, "generate", "today", "--config", configPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("generate current report: %v", err)
|
|
||||||
}
|
|
||||||
return decodeGenerateSummary(t, output.stdout)
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeGenerateSummary(t *testing.T, text string) generateSummary {
|
|
||||||
t.Helper()
|
|
||||||
var summary generateSummary
|
|
||||||
if err := json.Unmarshal([]byte(text), &summary); err != nil {
|
|
||||||
t.Fatalf("decode generate summary: %v\n%s", err, text)
|
|
||||||
}
|
|
||||||
return summary
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeBatchSummary(t *testing.T, text string) batchSummary {
|
|
||||||
t.Helper()
|
|
||||||
var summary batchSummary
|
|
||||||
if err := json.Unmarshal([]byte(text), &summary); err != nil {
|
|
||||||
t.Fatalf("decode batch summary: %v\n%s", err, text)
|
|
||||||
}
|
|
||||||
return summary
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertRoutineOutputSafe(t *testing.T, output commandOutput) {
|
|
||||||
t.Helper()
|
|
||||||
combined := output.stdout + output.stderr
|
|
||||||
for _, forbidden := range []string{testRenderedPrompt, testSchemaBody, testDataBody, testGeneratedBody, testEndpoint, testParameters, testCredential, "credential@example.invalid"} {
|
|
||||||
if strings.Contains(combined, forbidden) {
|
|
||||||
t.Fatalf("routine output contains sensitive value %q:\n%s", forbidden, combined)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type cliFixture struct {
|
|
||||||
tempDir string
|
|
||||||
workspaceRoot string
|
|
||||||
configPath string
|
|
||||||
}
|
|
||||||
|
|
||||||
func newCLIFixture(t *testing.T) cliFixture {
|
|
||||||
t.Helper()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
workspaceRoot := filepath.Join(tempDir, "workspace")
|
|
||||||
server := weatherServer(t)
|
|
||||||
return cliFixture{tempDir: tempDir, workspaceRoot: workspaceRoot, configPath: writeCLIConfig(t, workspaceRoot, server.URL+"/")}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f cliFixture) path(name string) string { return filepath.Join(f.tempDir, name) }
|
|
||||||
|
|
||||||
func writeCLIConfig(t *testing.T, workspaceRoot, baseURL string) string {
|
|
||||||
t.Helper()
|
|
||||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
|
||||||
body := "weather_api:\n timezone: America/Chicago\n"
|
|
||||||
if baseURL != "" {
|
|
||||||
body += " base_url: " + baseURL + "\n"
|
|
||||||
}
|
|
||||||
body += "workspace:\n root: " + workspaceRoot + "\n"
|
|
||||||
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
|
|
||||||
t.Fatalf("write config: %v", err)
|
|
||||||
}
|
|
||||||
return configPath
|
|
||||||
}
|
|
||||||
|
|
||||||
func weatherServer(t *testing.T) *httptest.Server {
|
|
||||||
t.Helper()
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/observations":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"timestamp":"2026-05-29T14:00:00Z","conditionCode":3}}`))
|
|
||||||
case "/conditions/current":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"conditionText":"Clear"}}`))
|
|
||||||
case "/forecast/hourly":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"locationId":"test-grid","locationName":"Testville","issuedAt":"2026-05-29T10:30:00-05:00","product":"hourly","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T07:00:00-05:00","textDescription":"Showers","temperatureF":66,"probabilityOfPrecipitationPercent":80},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T07:00:00-05:00","textDescription":"Showers","temperatureF":67,"probabilityOfPrecipitationPercent":70}]}}`))
|
|
||||||
case "/forecast/narrative":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"issuedAt":"2026-05-29T10:30:00-05:00","product":"narrative","periods":[{"startTime":"2026-05-29T06:00:00-05:00","endTime":"2026-05-29T18:00:00-05:00","textDescription":"Morning showers."},{"startTime":"2026-05-30T06:00:00-05:00","endTime":"2026-05-30T18:00:00-05:00","textDescription":"Tomorrow starts showery."}]}}`))
|
|
||||||
case "/alerts/active":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"alerts":[]}}`))
|
|
||||||
case "/discussion":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"product":"discussion","issuedAt":"2026-05-29T09:25:00-05:00","keyMessages":["Showers remain possible."]}}`))
|
|
||||||
case "/weatherstories/latest":
|
|
||||||
_, _ = w.Write([]byte(`{"data":null}`))
|
|
||||||
case "/outlooks/convective":
|
|
||||||
_, _ = w.Write([]byte(`{"data":{"asOf":"2026-05-29T16:00:00Z","outlooks":[],"discussions":[]}}`))
|
|
||||||
default:
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
return server
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeHistoricalInspectionFixture(t *testing.T, workspaceRoot string, reportID report.ID, generatedAt time.Time) string {
|
|
||||||
t.Helper()
|
|
||||||
runID := "historical-" + string(reportID)
|
|
||||||
date := generatedAt.Format(timeutil.DateLayout)
|
|
||||||
dir := filepath.Join(workspaceRoot, "snapshots", string(reportID), date)
|
|
||||||
modulePath := filepath.Join(dir, "modules."+runID+".json")
|
|
||||||
dataPath := filepath.Join(workspaceRoot, "data-packages", string(reportID), date, "data_package."+runID+".yaml")
|
|
||||||
metadataPath := filepath.Join(dir, "metadata."+runID+".json")
|
|
||||||
if err := os.MkdirAll(filepath.Dir(dataPath), 0o755); err != nil {
|
|
||||||
t.Fatalf("create historical fixture directory: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
t.Fatalf("create historical metadata directory: %v", err)
|
|
||||||
}
|
|
||||||
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": runID}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("build historical module snapshot: %v", err)
|
|
||||||
}
|
|
||||||
moduleData, err := json.Marshal(snapshot)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal historical module snapshot: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(modulePath, moduleData, 0o600); err != nil {
|
|
||||||
t.Fatalf("write historical module snapshot: %v", err)
|
|
||||||
}
|
|
||||||
period := timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}
|
|
||||||
pkg, err := promptinput.Build(promptinput.BuildRequest{Metadata: promptinput.Metadata{
|
|
||||||
RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID), GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period,
|
|
||||||
}, Modules: snapshot})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("build historical data package: %v", err)
|
|
||||||
}
|
|
||||||
data, err := promptinput.MarshalYAML(pkg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal historical data package: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(dataPath, data, 0o600); err != nil {
|
|
||||||
t.Fatalf("write historical data package: %v", err)
|
|
||||||
}
|
|
||||||
metadata := state.Metadata{
|
|
||||||
SchemaVersion: state.MetadataSchemaVersionV1, RunID: runID, ReportID: reportID, PromptID: "weather." + string(reportID),
|
|
||||||
GeneratedAt: generatedAt, Timezone: "UTC", ValidPeriod: period, SourceLocation: "historical archive",
|
|
||||||
ModuleSnapshotPath: modulePath, DataPackagePath: dataPath, PreflightPath: "/archive/preflight.json", GeneratedTextResultPath: "/archive/result.json",
|
|
||||||
}
|
|
||||||
metadataData, err := json.Marshal(metadata)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal historical metadata: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(metadataPath, metadataData, 0o600); err != nil {
|
|
||||||
t.Fatalf("write historical metadata: %v", err)
|
|
||||||
}
|
|
||||||
return runID
|
|
||||||
}
|
|
||||||
@@ -16,8 +16,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v1"
|
promptPreparationDebugSchemaVersion = "weatherreporter.prompt_preparation_debug.v2"
|
||||||
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v1"
|
promptExecutionDebugSchemaVersion = "weatherreporter.prompt_execution_debug.v2"
|
||||||
debugDirectoryMode = 0o700
|
debugDirectoryMode = 0o700
|
||||||
debugFileMode = 0o600
|
debugFileMode = 0o600
|
||||||
)
|
)
|
||||||
@@ -64,7 +64,6 @@ type PromptDebugPreparation struct {
|
|||||||
StartedAt time.Time `json:"startedAt"`
|
StartedAt time.Time `json:"startedAt"`
|
||||||
EndedAt time.Time `json:"endedAt"`
|
EndedAt time.Time `json:"endedAt"`
|
||||||
Duration time.Duration `json:"duration"`
|
Duration time.Duration `json:"duration"`
|
||||||
DataPackagePath string `json:"dataPackagePath"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptPreparationDebugArtifact is the on-disk preparation debug record.
|
// PromptPreparationDebugArtifact is the on-disk preparation debug record.
|
||||||
@@ -114,7 +113,6 @@ type PromptDebugExecution struct {
|
|||||||
StartedAt time.Time `json:"startedAt"`
|
StartedAt time.Time `json:"startedAt"`
|
||||||
EndedAt time.Time `json:"endedAt"`
|
EndedAt time.Time `json:"endedAt"`
|
||||||
Duration time.Duration `json:"duration"`
|
Duration time.Duration `json:"duration"`
|
||||||
DataPackagePath string `json:"dataPackagePath"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptExecutionDebugArtifact is the on-disk execution debug record.
|
// PromptExecutionDebugArtifact is the on-disk execution debug record.
|
||||||
@@ -340,7 +338,7 @@ func promptDebugPreparation(value promptexec.Preparation) PromptDebugPreparation
|
|||||||
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
|
RenderedPromptHash: value.RenderedPromptHash, InputHashes: copyPromptDebugMap(value.InputHashes),
|
||||||
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
|
ProfileID: value.ProfileID, BackendID: value.BackendID, ModelName: value.ModelName,
|
||||||
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
|
Output: PromptDebugOutput{Format: value.Output.Format, ValidationMode: value.Output.ValidationMode, SchemaPath: value.Output.SchemaPath},
|
||||||
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
|
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -351,7 +349,7 @@ func promptDebugExecution(value promptexec.Execution) PromptDebugExecution {
|
|||||||
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
|
InputHashes: copyPromptDebugMap(value.InputHashes), ProfileID: value.ProfileID,
|
||||||
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
|
BackendID: value.BackendID, ModelName: value.ModelName, GeneratedHash: value.GeneratedHash,
|
||||||
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
|
Usage: PromptDebugUsage{PromptTokens: value.Usage.PromptTokens, CompletionTokens: value.Usage.CompletionTokens, TotalTokens: value.Usage.TotalTokens, CachedTokens: value.Usage.CachedTokens, CacheWriteTokens: value.Usage.CacheWriteTokens},
|
||||||
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration, DataPackagePath: value.DataPackagePath,
|
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -40,19 +40,19 @@ func TestPromptDebugWriterWritesIsolatedArtifacts(t *testing.T) {
|
|||||||
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
|
t.Fatalf("WriteExecution() directory = %q, want %q", executionDir, preparationDir)
|
||||||
}
|
}
|
||||||
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
|
preparationData := readPromptDebugFile(t, filepath.Join(preparationDir, "preparation.json"))
|
||||||
for _, want := range []string{"Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
|
for _, want := range []string{"weatherreporter.prompt_preparation_debug.v2", "Use the supplied weather facts.", `"type": "object"`, "https://llm.example.test/v1/chat?api_key=%5Bredacted%5D", `"temperature": 0.2`, `"api_key": "[redacted]"`} {
|
||||||
if !strings.Contains(string(preparationData), want) {
|
if !strings.Contains(string(preparationData), want) {
|
||||||
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
|
t.Fatalf("preparation debug artifact missing %q:\n%s", want, preparationData)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
|
executionData := readPromptDebugFile(t, filepath.Join(executionDir, "execution.json"))
|
||||||
for _, want := range []string{"Generated forecast prose.", "validation details", `"status": "passed"`} {
|
for _, want := range []string{"weatherreporter.prompt_execution_debug.v2", "Generated forecast prose.", "validation details", `"status": "passed"`} {
|
||||||
if !strings.Contains(string(executionData), want) {
|
if !strings.Contains(string(executionData), want) {
|
||||||
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
|
t.Fatalf("execution debug artifact missing %q:\n%s", want, executionData)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, data := range [][]byte{preparationData, executionData} {
|
for _, data := range [][]byte{preparationData, executionData} {
|
||||||
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") {
|
if strings.Contains(string(data), "credential") || strings.Contains(string(data), "resolved-secret-value") || strings.Contains(string(data), "dataPackagePath") {
|
||||||
t.Fatalf("debug artifact contains credentials:\n%s", data)
|
t.Fatalf("debug artifact contains credentials:\n%s", data)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,7 +169,7 @@ func promptDebugPreparationFixture() promptexec.Preparation {
|
|||||||
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
PromptID: "weather.daily", PromptVersion: "v1", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash",
|
||||||
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
||||||
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
|
Output: promptexec.OutputContract{Format: "json_schema", ValidationMode: "strict", SchemaPath: "schemas/daily.json"},
|
||||||
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second, DataPackagePath: "/packages/daily.yaml",
|
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,7 +181,7 @@ func promptDebugExecutionFixture() promptexec.Execution {
|
|||||||
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
||||||
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
||||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
|
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
|
||||||
DataPackagePath: "/packages/daily.yaml", RawOutput: []byte("Generated forecast prose."),
|
RawOutput: []byte("Generated forecast prose."),
|
||||||
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,14 +61,12 @@ type ProfileInspection struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteRequest selects one exact prompt execution. DataPackage is the exact
|
// ExecuteRequest selects one exact prompt execution. DataPackage is the exact
|
||||||
// YAML input; implementations must copy it before retaining it. DataPackagePath
|
// YAML input; implementations must copy it before retaining it.
|
||||||
// is provenance for the inline input, not a provider-readable file reference.
|
|
||||||
type ExecuteRequest struct {
|
type ExecuteRequest struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
DataPackage []byte
|
DataPackage []byte
|
||||||
DataPackagePath string
|
|
||||||
CaptureDebug bool
|
CaptureDebug bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,7 +88,6 @@ type Preparation struct {
|
|||||||
StartedAt time.Time
|
StartedAt time.Time
|
||||||
EndedAt time.Time
|
EndedAt time.Time
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
DataPackagePath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreparationDebug contains content-rich preparation details for an explicitly
|
// PreparationDebug contains content-rich preparation details for an explicitly
|
||||||
@@ -127,7 +124,6 @@ type Execution struct {
|
|||||||
EndedAt time.Time
|
EndedAt time.Time
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
Validation Validation
|
Validation Validation
|
||||||
DataPackagePath string
|
|
||||||
RawOutput []byte
|
RawOutput []byte
|
||||||
Debug *ExecutionDebug
|
Debug *ExecutionDebug
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
|
|||||||
if executor.operationalFailure != nil {
|
if executor.operationalFailure != nil {
|
||||||
return nil, executor.operationalFailure
|
return nil, executor.operationalFailure
|
||||||
}
|
}
|
||||||
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath}
|
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion}
|
||||||
var debug *PreparationDebug
|
var debug *PreparationDebug
|
||||||
if request.CaptureDebug {
|
if request.CaptureDebug {
|
||||||
debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}}
|
debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}}
|
||||||
@@ -57,7 +57,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
|
|||||||
if executor.validationRejected {
|
if executor.validationRejected {
|
||||||
status = ValidationFailed
|
status = ValidationFailed
|
||||||
}
|
}
|
||||||
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, DataPackagePath: request.DataPackagePath, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
|
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
|
||||||
if request.CaptureDebug {
|
if request.CaptureDebug {
|
||||||
result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")}
|
result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")}
|
||||||
}
|
}
|
||||||
@@ -65,7 +65,7 @@ func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteReq
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutorLifecycleFixtures(t *testing.T) {
|
func TestExecutorLifecycleFixtures(t *testing.T) {
|
||||||
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", DataPackagePath: "data_package.yaml"}
|
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"}
|
||||||
t.Run("callback failure prevents provider execution", func(t *testing.T) {
|
t.Run("callback failure prevents provider execution", func(t *testing.T) {
|
||||||
executor := &lifecycleExecutor{}
|
executor := &lifecycleExecutor{}
|
||||||
callbackError := errors.New("persistence failed")
|
callbackError := errors.New("persistence failed")
|
||||||
@@ -239,7 +239,6 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
|
|||||||
BackendID: "local",
|
BackendID: "local",
|
||||||
ModelName: "model-name",
|
ModelName: "model-name",
|
||||||
Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"},
|
Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"},
|
||||||
DataPackagePath: "data-packages/daily/data_package.yaml",
|
|
||||||
}
|
}
|
||||||
execution := Execution{
|
execution := Execution{
|
||||||
RunID: "run-id",
|
RunID: "run-id",
|
||||||
@@ -254,7 +253,7 @@ func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
|
|||||||
GeneratedHash: "generated-hash",
|
GeneratedHash: "generated-hash",
|
||||||
RawOutput: []byte("generated content"),
|
RawOutput: []byte("generated content"),
|
||||||
}
|
}
|
||||||
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + preparation.DataPackagePath + execution.RunID + execution.GeneratedHash
|
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash
|
||||||
for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} {
|
for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} {
|
||||||
if strings.Contains(text, unwanted) {
|
if strings.Contains(text, unwanted) {
|
||||||
t.Fatalf("safe values contain %q: %s", unwanted, text)
|
t.Fatalf("safe values contain %q: %s", unwanted, text)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||||
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||||
@@ -366,21 +367,40 @@ func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) Promp
|
|||||||
artifact.DataPackagePath = paths.DataPackage
|
artifact.DataPackagePath = paths.DataPackage
|
||||||
artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
||||||
artifact.Preparation.PromptHash = "prompt-hash"
|
artifact.Preparation.PromptHash = "prompt-hash"
|
||||||
artifact.Preparation.DataPackagePath = paths.DataPackage
|
|
||||||
return artifact
|
return artifact
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validPreparationArtifact() PromptPreparationArtifact {
|
||||||
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
|
return PromptPreparationArtifact{
|
||||||
|
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
||||||
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
||||||
|
DataPackagePath: "/workspace/data.yaml", Preparation: &promptexec.Preparation{PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0"},
|
||||||
|
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact {
|
func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact {
|
||||||
artifact := validExecutionArtifact()
|
artifact := validExecutionArtifact()
|
||||||
metadata := resolved.Metadata()
|
metadata := resolved.Metadata()
|
||||||
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
|
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
|
||||||
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
|
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
|
||||||
artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
||||||
artifact.Provenance.DataPackagePath = paths.DataPackage
|
|
||||||
artifact.Paths.RawOutputPath = paths.GeneratedTextRaw
|
artifact.Paths.RawOutputPath = paths.GeneratedTextRaw
|
||||||
return artifact
|
return artifact
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validExecutionArtifact() PromptExecutionArtifact {
|
||||||
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||||
|
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
||||||
|
return PromptExecutionArtifact{
|
||||||
|
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
||||||
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
||||||
|
Provenance: &PromptExecutionProvenance{RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", BackendID: "backend", ModelName: "model", StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second},
|
||||||
|
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeJSONFixture(t *testing.T, path string, value any) {
|
func writeJSONFixture(t *testing.T, path string, value any) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := json.Marshal(value)
|
data, err := json.Marshal(value)
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func (a PromptPreparationArtifact) Validate() error {
|
|||||||
if a.Preparation == nil || a.Error != nil {
|
if a.Preparation == nil || a.Error != nil {
|
||||||
return fmt.Errorf("successful prompt preparation requires preparation without an error")
|
return fmt.Errorf("successful prompt preparation requires preparation without an error")
|
||||||
}
|
}
|
||||||
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath {
|
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion {
|
||||||
return fmt.Errorf("successful prompt preparation provenance must match the artifact")
|
return fmt.Errorf("successful prompt preparation provenance must match the artifact")
|
||||||
}
|
}
|
||||||
case PromptPreparationFailed:
|
case PromptPreparationFailed:
|
||||||
@@ -122,7 +122,6 @@ type PromptExecutionProvenance struct {
|
|||||||
StartedAt time.Time `json:"startedAt"`
|
StartedAt time.Time `json:"startedAt"`
|
||||||
EndedAt time.Time `json:"endedAt"`
|
EndedAt time.Time `json:"endedAt"`
|
||||||
Duration time.Duration `json:"duration"`
|
Duration time.Duration `json:"duration"`
|
||||||
DataPackagePath string `json:"dataPackagePath"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptExecutionPaths records only destinations reached by a completed run.
|
// PromptExecutionPaths records only destinations reached by a completed run.
|
||||||
@@ -165,7 +164,6 @@ func PromptExecutionProvenanceFrom(value promptexec.Execution) PromptExecutionPr
|
|||||||
InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID,
|
InputHashes: inputHashes, ProfileID: value.ProfileID, BackendID: value.BackendID,
|
||||||
ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage,
|
ModelName: value.ModelName, GeneratedHash: value.GeneratedHash, Usage: value.Usage,
|
||||||
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
StartedAt: value.StartedAt, EndedAt: value.EndedAt, Duration: value.Duration,
|
||||||
DataPackagePath: value.DataPackagePath,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,7 +253,6 @@ func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error {
|
|||||||
}{
|
}{
|
||||||
{"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash},
|
{"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash},
|
||||||
{"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName},
|
{"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName},
|
||||||
{"data package path", value.DataPackagePath},
|
|
||||||
} {
|
} {
|
||||||
if strings.TrimSpace(required.value) == "" {
|
if strings.TrimSpace(required.value) == "" {
|
||||||
return fmt.Errorf("completed prompt execution provenance %s is required", required.name)
|
return fmt.Errorf("completed prompt execution provenance %s is required", required.name)
|
||||||
|
|||||||
@@ -1,207 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
||||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPromptPreparationArtifactValidation(t *testing.T) {
|
|
||||||
valid := validPreparationArtifact()
|
|
||||||
if err := valid.Validate(); err != nil {
|
|
||||||
t.Fatalf("valid successful preparation: %v", err)
|
|
||||||
}
|
|
||||||
failed := valid
|
|
||||||
failed.Status = PromptPreparationFailed
|
|
||||||
failed.Preparation = nil
|
|
||||||
failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
|
||||||
if err := failed.Validate(); err != nil {
|
|
||||||
t.Fatalf("valid failed preparation: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(*PromptPreparationArtifact)
|
|
||||||
}{
|
|
||||||
{"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }},
|
|
||||||
{"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }},
|
|
||||||
{"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }},
|
|
||||||
{"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }},
|
|
||||||
{"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }},
|
|
||||||
{"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }},
|
|
||||||
{"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }},
|
|
||||||
{"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }},
|
|
||||||
{"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }},
|
|
||||||
{"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
|
||||||
{"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }},
|
|
||||||
{"success error", func(a *PromptPreparationArtifact) {
|
|
||||||
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
|
||||||
}},
|
|
||||||
{"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }},
|
|
||||||
{"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }},
|
|
||||||
{"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
artifact := validPreparationArtifact()
|
|
||||||
test.mutate(&artifact)
|
|
||||||
if err := artifact.Validate(); err == nil {
|
|
||||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(*PromptPreparationArtifact)
|
|
||||||
}{
|
|
||||||
{"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }},
|
|
||||||
{"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
|
||||||
{"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }},
|
|
||||||
{"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }},
|
|
||||||
{"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
artifact := validPreparationArtifact()
|
|
||||||
artifact.Status = PromptPreparationFailed
|
|
||||||
artifact.Preparation = nil
|
|
||||||
artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
|
||||||
test.mutate(&artifact)
|
|
||||||
if err := artifact.Validate(); err == nil {
|
|
||||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPromptExecutionArtifactValidation(t *testing.T) {
|
|
||||||
valid := validExecutionArtifact()
|
|
||||||
valid.Provenance.RunID = "provider-run-different-from-weatherreporter"
|
|
||||||
valid.Provenance.GeneratedHash = ""
|
|
||||||
valid.Provenance.Usage = promptexec.TokenUsage{}
|
|
||||||
if err := valid.Validate(); err != nil {
|
|
||||||
t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err)
|
|
||||||
}
|
|
||||||
rejected := validExecutionArtifact()
|
|
||||||
rejected.Status = PromptExecutionValidationRejected
|
|
||||||
validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"})
|
|
||||||
rejected.Validation = &validation
|
|
||||||
if err := rejected.Validate(); err != nil {
|
|
||||||
t.Fatalf("valid validation rejection: %v", err)
|
|
||||||
}
|
|
||||||
failed := validFailedExecutionArtifact()
|
|
||||||
if err := failed.Validate(); err != nil {
|
|
||||||
t.Fatalf("valid operational failure: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(*PromptExecutionArtifact)
|
|
||||||
}{
|
|
||||||
{"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }},
|
|
||||||
{"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }},
|
|
||||||
{"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }},
|
|
||||||
{"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }},
|
|
||||||
{"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }},
|
|
||||||
{"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }},
|
|
||||||
{"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }},
|
|
||||||
{"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
|
||||||
{"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }},
|
|
||||||
{"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }},
|
|
||||||
{"wrong validation", func(a *PromptExecutionArtifact) {
|
|
||||||
value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil)
|
|
||||||
a.Validation = &value
|
|
||||||
}},
|
|
||||||
{"operational error", func(a *PromptExecutionArtifact) {
|
|
||||||
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
|
||||||
}},
|
|
||||||
{"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }},
|
|
||||||
{"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }},
|
|
||||||
{"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }},
|
|
||||||
{"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }},
|
|
||||||
{"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }},
|
|
||||||
{"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }},
|
|
||||||
{"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }},
|
|
||||||
{"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }},
|
|
||||||
{"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }},
|
|
||||||
{"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }},
|
|
||||||
{"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }},
|
|
||||||
{"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }},
|
|
||||||
{"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
artifact := validExecutionArtifact()
|
|
||||||
test.mutate(&artifact)
|
|
||||||
if err := artifact.Validate(); err == nil {
|
|
||||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
mutate func(*PromptExecutionArtifact)
|
|
||||||
}{
|
|
||||||
{"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }},
|
|
||||||
{"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
|
||||||
{"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }},
|
|
||||||
{"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
artifact := validFailedExecutionArtifact()
|
|
||||||
test.mutate(&artifact)
|
|
||||||
if err := artifact.Validate(); err == nil {
|
|
||||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validPreparationArtifact() PromptPreparationArtifact {
|
|
||||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
return PromptPreparationArtifact{
|
|
||||||
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
|
|
||||||
PromptVersion: "2.0.0", DataPackagePath: "/workspace/data.yaml",
|
|
||||||
Preparation: &promptexec.Preparation{
|
|
||||||
PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
|
||||||
DataPackagePath: "/workspace/data.yaml",
|
|
||||||
},
|
|
||||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validExecutionArtifact() PromptExecutionArtifact {
|
|
||||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
|
||||||
return PromptExecutionArtifact{
|
|
||||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
|
||||||
Provenance: &PromptExecutionProvenance{
|
|
||||||
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
|
||||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
|
|
||||||
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
|
|
||||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
||||||
},
|
|
||||||
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validFailedExecutionArtifact() PromptExecutionArtifact {
|
|
||||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
||||||
return PromptExecutionArtifact{
|
|
||||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
|
|
||||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "2.0.0",
|
|
||||||
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user