From 236e3d16c4515239aac77554d3a0f0b50df0c3ca Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 2 Aug 2026 05:35:59 +0000 Subject: [PATCH] Separate report execution from publication --- internal/app/generation_test.go | 22 ++- internal/app/profile_execution.go | 127 ++++++++++++++++ internal/app/profile_execution_test.go | 70 +++++++++ internal/app/prompt_generate.go | 198 +++++++++---------------- 4 files changed, 290 insertions(+), 127 deletions(-) create mode 100644 internal/app/profile_execution.go create mode 100644 internal/app/profile_execution_test.go diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go index d086673..db19d8e 100644 --- a/internal/app/generation_test.go +++ b/internal/app/generation_test.go @@ -35,6 +35,7 @@ func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Re type generationExecutor struct { called bool + executeCalls int promptInspections int inspectErr error executeErr error @@ -61,6 +62,7 @@ func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRe return nil, err } e.called = true + e.executeCalls++ if e.executeErr != nil { return nil, e.executeErr } @@ -101,7 +103,7 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) { if err != nil { t.Fatalf("GenerateDetailed() error = %v", err) } - if !executor.called || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" { + if !executor.called || executor.executeCalls != 1 || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" { t.Fatalf("result = %#v", result) } if result.LLMDebugPath != "" { @@ -335,6 +337,24 @@ func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) { } } +func TestGenerateDetailedWritesRequestedPromptDebugArtifacts(t *testing.T) { + bundle := generationBundle(t) + debugRoot := t.TempDir() + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: generationConfig(), Report: ReportDaily, + Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), + WorkingDir: t.TempDir(), LLMDebugDir: debugRoot, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, + }) + if err != nil || result == nil || result.LLMDebugPath == "" { + t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err) + } + for _, name := range []string{"preparation.json", "execution.json"} { + if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, name)); statErr != nil { + t.Fatalf("debug artifact %q: %v", name, statErr) + } + } +} + func generationConfig() config.Config { cfg := config.Defaults() cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home" diff --git a/internal/app/profile_execution.go b/internal/app/profile_execution.go new file mode 100644 index 0000000..9b66b0e --- /dev/null +++ b/internal/app/profile_execution.go @@ -0,0 +1,127 @@ +package app + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" +) + +type profileExecutionRequest struct { + Prepared preparedReport + Prompt PromptInspectionResult + Profile promptexec.ProfileInspection + Executor promptexec.Executor + DebugWriter *promptdebug.PromptDebugWriter + DebugRef *promptdebug.PromptDebugRef +} + +type profileExecutionOutcome struct { + ProfileID string + BackendID string + ModelName string + ValidationStatus promptexec.ValidationStatus + LLMDebugPath string +} + +type profileExecutionError struct { + operation string + err error + callbackFailure bool +} + +func (e *profileExecutionError) Error() string { + return e.operation + ": " + e.err.Error() +} + +func (e *profileExecutionError) Unwrap() error { + return e.err +} + +func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) { + outcome := profileExecutionOutcome{ + ProfileID: req.Profile.ProfileID, + BackendID: req.Profile.BackendID, + ModelName: req.Profile.ModelName, + } + if req.Executor == nil { + return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)} + } + + callbackFailed := false + preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { + outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName + if req.DebugWriter == nil || !req.DebugWriter.Enabled() { + return nil + } + if req.DebugRef == nil { + callbackFailed = true + return promptDebugWriteError(fmt.Errorf("prompt debug reference is required")) + } + path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug) + if err != nil { + callbackFailed = true + return promptDebugWriteError(err) + } + outcome.LLMDebugPath = path + return nil + } + + captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled() + execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{ + PromptID: req.Prompt.PromptID, + PromptVersion: req.Prompt.PromptVersion, + ProfileID: req.Profile.ProfileID, + DataPackage: req.Prepared.dataPackageCopy(), + CaptureDebug: captureDebug, + }, preparationCallback) + if err != nil { + if callbackFailed { + return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true} + } + return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)} + } + if execution == nil { + return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)} + } + + outcome.ValidationStatus = execution.Validation.Status + if req.DebugWriter != nil && req.DebugWriter.Enabled() { + if req.DebugRef == nil { + return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))} + } + path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution) + if err != nil { + return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)} + } + if path != "" { + outcome.LLMDebugPath = path + } + } + + if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { + return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)} + } + if execution.Validation.Status == promptexec.ValidationFailed { + return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)} + } + + generatedText, _, err := req.Prepared.handler.Validate(execution.RawOutput) + if err != nil { + return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err} + } + metadata, snapshot, reportFacts, err := req.Prepared.renderInputs() + if err != nil { + return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err} + } + renderContext, err := req.Prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText) + if err != nil { + return outcome, nil, &profileExecutionError{operation: "build render context", err: err} + } + rendered, err := req.Prepared.handler.Render(renderContext) + if err != nil { + return outcome, nil, &profileExecutionError{operation: "render template", err: err} + } + return outcome, rendered, nil +} diff --git a/internal/app/profile_execution_test.go b/internal/app/profile_execution_test.go new file mode 100644 index 0000000..5afaba6 --- /dev/null +++ b/internal/app/profile_execution_test.go @@ -0,0 +1,70 @@ +package app + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" +) + +func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) { + prepared, inspection := preparedDailyProfile(t) + executor := &generationExecutor{} + outputPath := filepath.Join(t.TempDir(), "report.md") + outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{ + Prepared: prepared, Prompt: inspection, + Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName}, + Executor: executor, + }) + if err != nil { + t.Fatalf("executePreparedProfile() error = %v", err) + } + if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 { + t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls) + } + if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) { + t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr) + } +} + +func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) { + prepared, inspection := preparedDailyProfile(t) + debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir()) + if err != nil { + t.Fatalf("NewPromptDebugWriter() error = %v", err) + } + executor := &generationExecutor{} + outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{ + Prepared: prepared, Prompt: inspection, + Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName}, + Executor: executor, DebugWriter: debugWriter, + DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"}, + }) + var executionErr *profileExecutionError + if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" { + t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls) + } +} + +func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) { + t.Helper() + cfg := generationConfig() + resolved, err := ResolveGenerate(GenerateRequest{ + Config: cfg, Report: ReportDaily, + Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), + }, generationTime("2026-05-29T08:30:00-05:00")) + if err != nil { + t.Fatalf("ResolveGenerate() error = %v", err) + } + bundle := generationBundle(t) + prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}) + if err != nil { + t.Fatalf("prepareReport() error = %v", err) + } + return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"} +} diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index c7a0236..ab04498 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -22,47 +22,7 @@ type promptReportRequest struct { noNotify bool } -type promptReportWorkflow struct { - ctx context.Context - req promptReportRequest - result *ReportResult - prepared preparedReport - debugRef promptdebug.PromptDebugRef - callbackFailed bool -} - func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) { - workflow, err := newPromptReportWorkflow(ctx, req) - if err != nil { - return nil, err - } - if err := workflow.buildInputs(); err != nil { - return workflow.result, err - } - execution, err := workflow.executePrompt() - if err != nil { - if workflow.callbackFailed { - return workflow.result, err - } - return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err)) - } - if execution == nil { - return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)) - } - workflow.result.ValidationStatus = execution.Validation.Status - if err := workflow.writeExecutionDebug(*execution); err != nil { - return workflow.result, err - } - if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed { - return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)) - } - if execution.Validation.Status == promptexec.ValidationFailed { - return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)) - } - return workflow.renderAndPublish(execution.RawOutput) -} - -func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) { if req.Collection.Bundle == nil { return nil, fmt.Errorf("collected weather bundle is required") } @@ -70,10 +30,36 @@ func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*pro if result == nil { result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection) } - return &promptReportWorkflow{ - ctx: ctx, req: req, - result: result, - }, nil + prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection}) + if err != nil { + return result, generatedPreparationError(req.Resolved, result.RunID, err) + } + result.SourceWarnings = prepared.sourceWarningsCopy() + debugRef := promptdebug.PromptDebugRef{ReportID: result.ReportID, ValidDate: prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: result.RunID} + outcome, rendered, err := executePreparedProfile(ctx, profileExecutionRequest{ + Prepared: prepared, + Prompt: req.Inspection, + Profile: promptexec.ProfileInspection{ + ProfileID: req.Inspection.ProfileID, + BackendID: req.Inspection.BackendID, + ModelName: req.Inspection.ModelName, + }, + Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef, + }) + result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName + result.ValidationStatus = outcome.ValidationStatus + result.LLMDebugPath = outcome.LLMDebugPath + if err != nil { + return result, generatedProfileExecutionError(req.Resolved, result.RunID, err) + } + return publishPromptReport(ctx, promptPublicationRequest{ + GenerateRequest: req.GenerateRequest, + Resolved: req.Resolved, + OutputPath: req.OutputPath, + Result: result, + Markdown: rendered, + suppressNotification: req.noNotify, + }) } func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult { @@ -87,91 +73,51 @@ func initialReportResult(req GenerateRequest, resolved report.Resolved, inspecti } } -func (w *promptReportWorkflow) buildInputs() error { - prepared, err := prepareReport(prepareReportRequest{Config: w.req.Config, Resolved: w.req.Resolved, Collection: w.req.Collection}) +type promptPublicationRequest struct { + GenerateRequest + Resolved report.Resolved + OutputPath string + Result *ReportResult + Markdown []byte + suppressNotification bool +} + +func publishPromptReport(ctx context.Context, req promptPublicationRequest) (*ReportResult, error) { + if err := publicationContextError(ctx); err != nil { + return req.Result, generatedReportError(req.Resolved, req.Result.RunID, "publish report", err) + } + if err := fileutil.WriteFileAtomic(req.OutputPath, req.Markdown); err != nil { + return req.Result, err + } + req.Result.OutputPath = req.OutputPath + if req.suppressNotification { + return req.Result, nil + } + notification, err := notifyReport(ctx, req.Config, req.Resolved, req.Result.OutputPath, req.Result.RunID, req.Result.GeneratedAt, req.Notifier) + req.Result.Notification = notification if err != nil { - var preparation *preparationError - if errors.As(err, &preparation) { - return w.reportError(preparation.operation, preparation.err) + return req.Result, err + } + return req.Result, nil +} + +func generatedPreparationError(resolved report.Resolved, runID string, err error) error { + var preparation *preparationError + if errors.As(err, &preparation) { + return generatedReportError(resolved, runID, preparation.operation, preparation.err) + } + return generatedReportError(resolved, runID, "prepare report", err) +} + +func generatedProfileExecutionError(resolved report.Resolved, runID string, err error) error { + var execution *profileExecutionError + if errors.As(err, &execution) { + if execution.callbackFailure { + return execution.err } - return w.reportError("prepare report", err) + return generatedReportError(resolved, runID, execution.operation, execution.err) } - w.prepared = prepared - w.result.SourceWarnings = w.prepared.sourceWarningsCopy() - w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID} - return nil -} - -func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) { - captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled() - return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.prepared.dataPackageCopy(), CaptureDebug: captureDebug}, w.writePreparationDebug) -} - -func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error { - w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName - if w.req.DebugWriter == nil { - return nil - } - path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug) - if err != nil { - w.callbackFailed = true - return promptDebugWriteError(err) - } - w.result.LLMDebugPath = path - return nil -} - -func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error { - if w.req.DebugWriter == nil { - return nil - } - path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution) - if err != nil { - return w.reportError("write prompt debug", promptDebugWriteError(err)) - } - if path != "" { - w.result.LLMDebugPath = path - } - return nil -} - -func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) { - generatedText, _, err := w.prepared.handler.Validate(raw) - if err != nil { - return w.result, w.reportError("validate generated text", err) - } - metadata, snapshot, reportFacts, err := w.prepared.renderInputs() - if err != nil { - return w.result, w.reportError("copy prepared render inputs", err) - } - renderContext, err := w.prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText) - if err != nil { - return w.result, w.reportError("build render context", err) - } - rendered, err := w.prepared.handler.Render(renderContext) - if err != nil { - return w.result, w.reportError("render template", err) - } - if err := publicationContextError(w.ctx); err != nil { - return w.result, w.reportError("publish report", err) - } - if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil { - return w.result, err - } - w.result.OutputPath = w.req.OutputPath - if w.req.noNotify { - return w.result, nil - } - notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier) - w.result.Notification = notification - if err != nil { - return w.result, err - } - return w.result, nil -} - -func (w *promptReportWorkflow) reportError(operation string, err error) error { - return generatedReportError(w.req.Resolved, w.result.RunID, operation, err) + return generatedReportError(resolved, runID, "execute prompt", err) } func classifiedPromptError(operation string, err error) error {