eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Separate report execution from publication

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-02 05:35:59 +00:00
parent 4fa873983d
commit 236e3d16c4
4 changed files with 290 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/app/generation_test.go
View File

@@ -35,6 +35,7 @@ func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Re
type generationExecutor struct {
called bool
executeCalls int
promptInspections int
inspectErr error
executeErr error
@@ -61,6 +62,7 @@ func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRe
return nil, err
}
e.called = true
e.executeCalls++
if e.executeErr != nil {
return nil, e.executeErr
}
@@ -101,7 +103,7 @@ func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if !executor.called || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
if !executor.called || executor.executeCalls != 1 || collector.calls != 1 || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
t.Fatalf("result = %#v", result)
}
if result.LLMDebugPath != "" {
@@ -335,6 +337,24 @@ func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) {
}
}
func TestGenerateDetailedWritesRequestedPromptDebugArtifacts(t *testing.T) {
bundle := generationBundle(t)
debugRoot := t.TempDir()
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: generationConfig(), Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
WorkingDir: t.TempDir(), LLMDebugDir: debugRoot, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
})
if err != nil || result == nil || result.LLMDebugPath == "" {
t.Fatalf("GenerateDetailed() result/error = %#v/%v", result, err)
}
for _, name := range []string{"preparation.json", "execution.json"} {
if _, statErr := os.Stat(filepath.Join(result.LLMDebugPath, name)); statErr != nil {
t.Fatalf("debug artifact %q: %v", name, statErr)
}
}
}
func generationConfig() config.Config {
cfg := config.Defaults()
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"

127
internal/app/profile_execution.go Normal file
View File

@@ -0,0 +1,127 @@
package app
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
type profileExecutionRequest struct {
Prepared preparedReport
Prompt PromptInspectionResult
Profile promptexec.ProfileInspection
Executor promptexec.Executor
DebugWriter *promptdebug.PromptDebugWriter
DebugRef *promptdebug.PromptDebugRef
}
type profileExecutionOutcome struct {
ProfileID string
BackendID string
ModelName string
ValidationStatus promptexec.ValidationStatus
LLMDebugPath string
}
type profileExecutionError struct {
operation string
err error
callbackFailure bool
}
func (e *profileExecutionError) Error() string {
return e.operation + ": " + e.err.Error()
}
func (e *profileExecutionError) Unwrap() error {
return e.err
}
func executePreparedProfile(ctx context.Context, req profileExecutionRequest) (profileExecutionOutcome, []byte, error) {
outcome := profileExecutionOutcome{
ProfileID: req.Profile.ProfileID,
BackendID: req.Profile.BackendID,
ModelName: req.Profile.ModelName,
}
if req.Executor == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)}
}
callbackFailed := false
preparationCallback := func(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
outcome.ProfileID, outcome.BackendID, outcome.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
if req.DebugWriter == nil || !req.DebugWriter.Enabled() {
return nil
}
if req.DebugRef == nil {
callbackFailed = true
return promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))
}
path, err := req.DebugWriter.WritePreparation(*req.DebugRef, preparation, debug)
if err != nil {
callbackFailed = true
return promptDebugWriteError(err)
}
outcome.LLMDebugPath = path
return nil
}
captureDebug := req.DebugWriter != nil && req.DebugWriter.Enabled()
execution, err := req.Executor.Execute(ctx, promptexec.ExecuteRequest{
PromptID: req.Prompt.PromptID,
PromptVersion: req.Prompt.PromptVersion,
ProfileID: req.Profile.ProfileID,
DataPackage: req.Prepared.dataPackageCopy(),
CaptureDebug: captureDebug,
}, preparationCallback)
if err != nil {
if callbackFailed {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: err, callbackFailure: true}
}
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: classifiedPromptError("prompt execution failed", err)}
}
if execution == nil {
return outcome, nil, &profileExecutionError{operation: "execute prompt", err: promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil)}
}
outcome.ValidationStatus = execution.Validation.Status
if req.DebugWriter != nil && req.DebugWriter.Enabled() {
if req.DebugRef == nil {
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(fmt.Errorf("prompt debug reference is required"))}
}
path, err := req.DebugWriter.WriteExecution(*req.DebugRef, *execution)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "write prompt debug", err: promptDebugWriteError(err)}
}
if path != "" {
outcome.LLMDebugPath = path
}
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil)}
}
if execution.Validation.Status == promptexec.ValidationFailed {
return outcome, nil, &profileExecutionError{operation: "validate prompt execution", err: promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil)}
}
generatedText, _, err := req.Prepared.handler.Validate(execution.RawOutput)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "validate generated text", err: err}
}
metadata, snapshot, reportFacts, err := req.Prepared.renderInputs()
if err != nil {
return outcome, nil, &profileExecutionError{operation: "copy prepared render inputs", err: err}
}
renderContext, err := req.Prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "build render context", err: err}
}
rendered, err := req.Prepared.handler.Render(renderContext)
if err != nil {
return outcome, nil, &profileExecutionError{operation: "render template", err: err}
}
return outcome, rendered, nil
}

70
internal/app/profile_execution_test.go Normal file
View File

@@ -0,0 +1,70 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
)
func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
executor := &generationExecutor{}
outputPath := filepath.Join(t.TempDir(), "report.md")
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor,
})
if err != nil {
t.Fatalf("executePreparedProfile() error = %v", err)
}
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
}
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr)
}
}
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
prepared, inspection := preparedDailyProfile(t)
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
if err != nil {
t.Fatalf("NewPromptDebugWriter() error = %v", err)
}
executor := &generationExecutor{}
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
Prepared: prepared, Prompt: inspection,
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
Executor: executor, DebugWriter: debugWriter,
DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"},
})
var executionErr *profileExecutionError
if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" {
t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls)
}
}
func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) {
t.Helper()
cfg := generationConfig()
resolved, err := ResolveGenerate(GenerateRequest{
Config: cfg, Report: ReportDaily,
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
}, generationTime("2026-05-29T08:30:00-05:00"))
if err != nil {
t.Fatalf("ResolveGenerate() error = %v", err)
}
bundle := generationBundle(t)
prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}})
if err != nil {
t.Fatalf("prepareReport() error = %v", err)
}
return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: "prompt-hash", ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"}
}

198
internal/app/prompt_generate.go
View File

@@ -22,47 +22,7 @@ type promptReportRequest struct {
noNotify bool
}
type promptReportWorkflow struct {
ctx context.Context
req promptReportRequest
result *ReportResult
prepared preparedReport
debugRef promptdebug.PromptDebugRef
callbackFailed bool
}
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
workflow, err := newPromptReportWorkflow(ctx, req)
if err != nil {
return nil, err
}
if err := workflow.buildInputs(); err != nil {
return workflow.result, err
}
execution, err := workflow.executePrompt()
if err != nil {
if workflow.callbackFailed {
return workflow.result, err
}
return workflow.result, workflow.reportError("execute prompt", classifiedPromptError("prompt execution failed", err))
}
if execution == nil {
return workflow.result, workflow.reportError("execute prompt", promptexec.NewError(promptexec.Generation, "prompt executor returned no execution", nil))
}
workflow.result.ValidationStatus = execution.Validation.Status
if err := workflow.writeExecutionDebug(*execution); err != nil {
return workflow.result, err
}
if execution.Validation.Status != promptexec.ValidationPassed && execution.Validation.Status != promptexec.ValidationFailed {
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.OperationalValidation, "prompt execution did not complete validation", nil))
}
if execution.Validation.Status == promptexec.ValidationFailed {
return workflow.result, workflow.reportError("validate prompt execution", promptexec.NewError(promptexec.ValidationRejected, "prompt output did not satisfy its schema", nil))
}
return workflow.renderAndPublish(execution.RawOutput)
}
func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*promptReportWorkflow, error) {
if req.Collection.Bundle == nil {
return nil, fmt.Errorf("collected weather bundle is required")
}
@@ -70,10 +30,36 @@ func newPromptReportWorkflow(ctx context.Context, req promptReportRequest) (*pro
if result == nil {
result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection)
}
return &promptReportWorkflow{
ctx: ctx, req: req,
result: result,
}, nil
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection})
if err != nil {
return result, generatedPreparationError(req.Resolved, result.RunID, err)
}
result.SourceWarnings = prepared.sourceWarningsCopy()
debugRef := promptdebug.PromptDebugRef{ReportID: result.ReportID, ValidDate: prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: result.RunID}
outcome, rendered, err := executePreparedProfile(ctx, profileExecutionRequest{
Prepared: prepared,
Prompt: req.Inspection,
Profile: promptexec.ProfileInspection{
ProfileID: req.Inspection.ProfileID,
BackendID: req.Inspection.BackendID,
ModelName: req.Inspection.ModelName,
},
Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef,
})
result.ProfileID, result.BackendID, result.ModelName = outcome.ProfileID, outcome.BackendID, outcome.ModelName
result.ValidationStatus = outcome.ValidationStatus
result.LLMDebugPath = outcome.LLMDebugPath
if err != nil {
return result, generatedProfileExecutionError(req.Resolved, result.RunID, err)
}
return publishPromptReport(ctx, promptPublicationRequest{
GenerateRequest: req.GenerateRequest,
Resolved: req.Resolved,
OutputPath: req.OutputPath,
Result: result,
Markdown: rendered,
suppressNotification: req.noNotify,
})
}
func initialReportResult(req GenerateRequest, resolved report.Resolved, inspection PromptInspectionResult) *ReportResult {
@@ -87,91 +73,51 @@ func initialReportResult(req GenerateRequest, resolved report.Resolved, inspecti
}
}
func (w *promptReportWorkflow) buildInputs() error {
prepared, err := prepareReport(prepareReportRequest{Config: w.req.Config, Resolved: w.req.Resolved, Collection: w.req.Collection})
type promptPublicationRequest struct {
GenerateRequest
Resolved report.Resolved
OutputPath string
Result *ReportResult
Markdown []byte
suppressNotification bool
}
func publishPromptReport(ctx context.Context, req promptPublicationRequest) (*ReportResult, error) {
if err := publicationContextError(ctx); err != nil {
return req.Result, generatedReportError(req.Resolved, req.Result.RunID, "publish report", err)
}
if err := fileutil.WriteFileAtomic(req.OutputPath, req.Markdown); err != nil {
return req.Result, err
}
req.Result.OutputPath = req.OutputPath
if req.suppressNotification {
return req.Result, nil
}
notification, err := notifyReport(ctx, req.Config, req.Resolved, req.Result.OutputPath, req.Result.RunID, req.Result.GeneratedAt, req.Notifier)
req.Result.Notification = notification
if err != nil {
var preparation *preparationError
if errors.As(err, &preparation) {
return w.reportError(preparation.operation, preparation.err)
return req.Result, err
}
return req.Result, nil
}
func generatedPreparationError(resolved report.Resolved, runID string, err error) error {
var preparation *preparationError
if errors.As(err, &preparation) {
return generatedReportError(resolved, runID, preparation.operation, preparation.err)
}
return generatedReportError(resolved, runID, "prepare report", err)
}
func generatedProfileExecutionError(resolved report.Resolved, runID string, err error) error {
var execution *profileExecutionError
if errors.As(err, &execution) {
if execution.callbackFailure {
return execution.err
}
return w.reportError("prepare report", err)
return generatedReportError(resolved, runID, execution.operation, execution.err)
}
w.prepared = prepared
w.result.SourceWarnings = w.prepared.sourceWarningsCopy()
w.debugRef = promptdebug.PromptDebugRef{ReportID: w.result.ReportID, ValidDate: w.prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: w.result.RunID}
return nil
}
func (w *promptReportWorkflow) executePrompt() (*promptexec.Execution, error) {
captureDebug := w.req.DebugWriter != nil && w.req.DebugWriter.Enabled()
return w.req.Executor.Execute(w.ctx, promptexec.ExecuteRequest{PromptID: w.req.Inspection.PromptID, PromptVersion: w.req.Inspection.PromptVersion, ProfileID: w.req.Inspection.ProfileID, DataPackage: w.prepared.dataPackageCopy(), CaptureDebug: captureDebug}, w.writePreparationDebug)
}
func (w *promptReportWorkflow) writePreparationDebug(preparation promptexec.Preparation, debug *promptexec.PreparationDebug) error {
w.result.ProfileID, w.result.BackendID, w.result.ModelName = preparation.ProfileID, preparation.BackendID, preparation.ModelName
if w.req.DebugWriter == nil {
return nil
}
path, err := w.req.DebugWriter.WritePreparation(w.debugRef, preparation, debug)
if err != nil {
w.callbackFailed = true
return promptDebugWriteError(err)
}
w.result.LLMDebugPath = path
return nil
}
func (w *promptReportWorkflow) writeExecutionDebug(execution promptexec.Execution) error {
if w.req.DebugWriter == nil {
return nil
}
path, err := w.req.DebugWriter.WriteExecution(w.debugRef, execution)
if err != nil {
return w.reportError("write prompt debug", promptDebugWriteError(err))
}
if path != "" {
w.result.LLMDebugPath = path
}
return nil
}
func (w *promptReportWorkflow) renderAndPublish(raw []byte) (*ReportResult, error) {
generatedText, _, err := w.prepared.handler.Validate(raw)
if err != nil {
return w.result, w.reportError("validate generated text", err)
}
metadata, snapshot, reportFacts, err := w.prepared.renderInputs()
if err != nil {
return w.result, w.reportError("copy prepared render inputs", err)
}
renderContext, err := w.prepared.handler.BuildRenderContext(metadata, snapshot, reportFacts.Collected, reportFacts.Derived, generatedText)
if err != nil {
return w.result, w.reportError("build render context", err)
}
rendered, err := w.prepared.handler.Render(renderContext)
if err != nil {
return w.result, w.reportError("render template", err)
}
if err := publicationContextError(w.ctx); err != nil {
return w.result, w.reportError("publish report", err)
}
if err := fileutil.WriteFileAtomic(w.req.OutputPath, rendered); err != nil {
return w.result, err
}
w.result.OutputPath = w.req.OutputPath
if w.req.noNotify {
return w.result, nil
}
notification, err := notifyReport(w.ctx, w.req.Config, w.req.Resolved, w.result.OutputPath, w.result.RunID, w.result.GeneratedAt, w.req.Notifier)
w.result.Notification = notification
if err != nil {
return w.result, err
}
return w.result, nil
}
func (w *promptReportWorkflow) reportError(operation string, err error) error {
return generatedReportError(w.req.Resolved, w.result.RunID, operation, err)
return generatedReportError(resolved, runID, "execute prompt", err)
}
func classifiedPromptError(operation string, err error) error {