eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Track completed execution artifact paths

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 16:51:12 +00:00
parent 870b54a4a0
commit 34c395d7e5
5 changed files with 337 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/internal/app-orchestration.md
View File

@@ -13,6 +13,11 @@ callback, executes the prepared prompt, saves execution provenance and raw
output, validates generated text, renders Markdown, and optionally copies or
notifies from the managed report.
After a completed prompt run, each successfully written downstream artifact is
atomically added to the execution record before the corresponding metadata
rewrite. Later failures therefore leave the original Promptkit outcome and its
last durable set of reached paths inspectable.
Failure results retain all safe paths reached so far. Validation rejection
persists raw output and execution provenance but does not render a report.

6
docs/internal/state.md
View File

@@ -37,6 +37,12 @@ consistent provenance, and status-appropriate validation or bounded classified
errors. Completed execution provenance keeps Promptkit's run identity distinct
from the Weatherreporter run identity.
For a completed prompt run, the execution record is atomically replaced after
each downstream artifact is saved. Its path set therefore records the raw and
normalized generated text, render context, managed report, requested output
copy, and notification artifact actually reached without changing the original
Promptkit outcome.
`PromptDebugWriter` is separate from workspace state. An empty root disables
it. An enabled absolute root is checked for safe directories and symlinks, then
stores `preparation.json` and `execution.json` beneath

20
internal/app/app.go
View File

@@ -534,6 +534,7 @@ type finalizeRenderedReportRequest struct {
Resolved report.Resolved
Metadata state.Metadata
MetadataPath string
ExecutionArtifact *state.PromptExecutionArtifact
ManagedReportPath string
OutputPath string
Notifier Notifier
@@ -556,15 +557,24 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque
if req.ManagedReportPath == "" {
return finalizeRenderedReportResult{}, fmt.Errorf("managed report path is required for report %q", req.Resolved.Definition.ID)
}
if req.ExecutionArtifact == nil {
return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID)
}
result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath}
if req.OutputPath != "" {
if req.GenerationErr == nil && req.OutputPath != req.ManagedReportPath {
if req.OutputPath != "" && req.GenerationErr == nil {
if req.OutputPath != req.ManagedReportPath {
if err := fileutil.CopyFileAtomic(req.ManagedReportPath, req.OutputPath); err != nil {
return result, err
}
result.OutputPath = req.OutputPath
req.ExecutionArtifact.Paths.OutputPath = req.OutputPath
if _, err := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); err != nil {
return result, err
}
} else {
result.OutputPath = req.OutputPath
}
result.OutputPath = req.OutputPath
}
metadata := req.Metadata
@@ -588,6 +598,10 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque
result.Notification = notification
metadata.NotificationPath = notificationPath
result.Metadata = metadata
req.ExecutionArtifact.Paths.NotificationPath = notificationPath
if _, saveErr := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); saveErr != nil {
return result, saveErr
}
metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata)
if saveErr != nil {
return result, saveErr

310
internal/app/prompt_artifact_paths_test.go
View File

@@ -6,6 +6,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -18,24 +19,60 @@ import (
)
const (
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failGeneratedText = "generated text"
failRenderContext = "render context"
failRenderedReportPath = "rendered report path"
failDistributorNotification = "distributor notification"
)
type failingPersistenceStore struct {
state.Store
failOperation string
failMetadataCall int
metadataCalls int
failOperation string
failExecutionCall int
failMetadataCall int
executionCalls int
metadataCalls int
renderedReportPath string
}
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
if s.failOperation == failPromptExecution {
s.executionCalls++
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
return "", errors.New("injected prompt execution persistence failure")
}
return s.Store.SavePromptExecution(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
if s.failOperation == failGeneratedText {
return "", errors.New("injected generated text persistence failure")
}
return s.Store.SaveGeneratedText(ctx, resolved, data)
}
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
if s.failOperation == failRenderContext {
return "", errors.New("injected render context persistence failure")
}
return s.Store.SaveRenderContext(ctx, resolved, value)
}
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
if s.failOperation == failRenderedReportPath {
return s.renderedReportPath, nil
}
return s.Store.PrepareRenderedReport(ctx, resolved)
}
func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) {
if s.failOperation == failDistributorNotification {
return "", errors.New("injected notification persistence failure")
}
return s.Store.SaveDistributorNotification(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
s.metadataCalls++
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
@@ -93,6 +130,12 @@ func (successfulNotifier) Notify(context.Context, NotificationRequest) (*Notific
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
}
type failingNotifier struct{}
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return nil, errors.New("injected notification failure")
}
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
tests := []struct {
name string
@@ -123,6 +166,7 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
@@ -138,9 +182,12 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
tests := []struct {
name string
executor artifactPathExecutor
want reachedPromptArtifacts
name string
executor artifactPathExecutor
want reachedPromptArtifacts
wantExecutionStatus state.PromptExecutionStatus
wantExecutionPaths state.PromptExecutionPaths
wantRawExecution bool
}{
{
name: "preparation failure",
@@ -148,29 +195,262 @@ func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
want: reachedPromptArtifacts{preparation: true, metadata: true},
},
{
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
wantExecutionStatus: state.PromptExecutionFailed,
},
{
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
wantExecutionStatus: state.PromptExecutionValidationRejected,
wantRawExecution: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, test.executor)
if test.wantRawExecution {
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
if test.wantExecutionStatus != "" {
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if loadErr != nil {
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
}
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
}
}
})
}
}
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
result, err := generatePromptReport(context.Background(), req)
if err != nil {
t.Fatalf("generatePromptReport() error = %v", err)
}
want := state.PromptExecutionPaths{
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
OutputPath: paths.output, NotificationPath: paths.Notification,
}
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
data, err := os.ReadFile(result.ExecutionPath)
if err != nil {
t.Fatalf("read execution artifact: %v", err)
}
text := string(data)
for _, forbidden := range []string{
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
"credential", "secret-value",
} {
if strings.Contains(text, forbidden) {
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
}
}
}
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
tests := []struct {
name string
failOperation string
failExecutionCall int
failMetadataCall int
requestOutput bool
failOutputCopy bool
notify bool
notificationFailure bool
wantExecution reachedExecutionArtifacts
wantResult reachedPromptArtifacts
}{
{
name: "normalized text write", failOperation: failGeneratedText,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
},
{
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context write", failOperation: failRenderContext,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report write", failOperation: failRenderedReportPath,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy write", requestOutput: true, failOutputCopy: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{
Store: req.Store, failOperation: test.failOperation,
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
}
if test.failOperation == failRenderedReportPath {
store.renderedReportPath = t.TempDir()
}
req.Store = store
if test.requestOutput {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.failOutputCopy {
blocker := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
t.Fatalf("write output blocker: %v", err)
}
req.OutputPath = filepath.Join(blocker, "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
if test.notificationFailure {
req.Notifier = failingNotifier{}
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
})
}
}
type reachedExecutionArtifacts struct {
raw bool
normalized bool
renderContext bool
report bool
output bool
notification bool
}
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
result := state.PromptExecutionPaths{}
if reached.raw {
result.RawOutputPath = paths.GeneratedTextRaw
}
if reached.normalized {
result.GeneratedTextPath = paths.GeneratedText
}
if reached.renderContext {
result.RenderContextPath = paths.RenderContext
}
if reached.report {
result.RenderedReportPath = paths.RenderedReport
}
if reached.output {
result.OutputPath = paths.output
}
if reached.notification {
result.NotificationPath = paths.Notification
}
return result
}
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
t.Helper()
artifact, err := store.LoadPromptExecution(context.Background(), path)
if err != nil {
t.Fatalf("LoadPromptExecution() error = %v", err)
}
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
}
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
}
if artifact.Paths != want {
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
}
}
type promptArtifactPaths struct {
state.ArtifactPaths
output string

15
internal/app/prompt_generate.go
View File

@@ -287,6 +287,10 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
metadata.GeneratedTextPath = generatedTextPath
result.GeneratedTextPath = generatedTextPath
result.Metadata = metadata
executionArtifact.Paths.GeneratedTextPath = generatedTextPath
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
return result, err
}
metadataPath, err = store.SaveMetadata(ctx, metadata)
if err != nil {
return result, err
@@ -304,6 +308,10 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
metadata.RenderContextPath = renderContextPath
result.RenderContextPath = renderContextPath
result.Metadata = metadata
executionArtifact.Paths.RenderContextPath = renderContextPath
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
return result, err
}
metadataPath, err = store.SaveMetadata(ctx, metadata)
if err != nil {
return result, err
@@ -324,9 +332,14 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
result.ReportPath = reportPath
metadata.RenderedReportPath = reportPath
result.Metadata = metadata
executionArtifact.Paths.RenderedReportPath = reportPath
if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil {
return result, err
}
finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{
Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, MetadataPath: result.MetadataPath,
ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier, noNotify: req.noNotify,
ExecutionArtifact: &executionArtifact, ManagedReportPath: reportPath, OutputPath: req.OutputPath,
Notifier: req.Notifier, noNotify: req.noNotify,
})
result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath
result.Metadata, result.MetadataPath, result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification