diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index 4e1ecde..979ad6b 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -13,6 +13,11 @@ callback, executes the prepared prompt, saves execution provenance and raw output, validates generated text, renders Markdown, and optionally copies or notifies from the managed report. +After a completed prompt run, each successfully written downstream artifact is +atomically added to the execution record before the corresponding metadata +rewrite. Later failures therefore leave the original Promptkit outcome and its +last durable set of reached paths inspectable. + Failure results retain all safe paths reached so far. Validation rejection persists raw output and execution provenance but does not render a report. diff --git a/docs/internal/state.md b/docs/internal/state.md index 5abcbaa..4619dcc 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -37,6 +37,12 @@ consistent provenance, and status-appropriate validation or bounded classified errors. Completed execution provenance keeps Promptkit's run identity distinct from the Weatherreporter run identity. +For a completed prompt run, the execution record is atomically replaced after +each downstream artifact is saved. Its path set therefore records the raw and +normalized generated text, render context, managed report, requested output +copy, and notification artifact actually reached without changing the original +Promptkit outcome. + `PromptDebugWriter` is separate from workspace state. An empty root disables it. An enabled absolute root is checked for safe directories and symlinks, then stores `preparation.json` and `execution.json` beneath diff --git a/internal/app/app.go b/internal/app/app.go index d28052a..f206c47 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -534,6 +534,7 @@ type finalizeRenderedReportRequest struct { Resolved report.Resolved Metadata state.Metadata MetadataPath string + ExecutionArtifact *state.PromptExecutionArtifact ManagedReportPath string OutputPath string Notifier Notifier @@ -556,15 +557,24 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque if req.ManagedReportPath == "" { return finalizeRenderedReportResult{}, fmt.Errorf("managed report path is required for report %q", req.Resolved.Definition.ID) } + if req.ExecutionArtifact == nil { + return finalizeRenderedReportResult{}, fmt.Errorf("prompt execution artifact is required for report %q", req.Resolved.Definition.ID) + } result := finalizeRenderedReportResult{Metadata: req.Metadata, MetadataPath: req.MetadataPath} - if req.OutputPath != "" { - if req.GenerationErr == nil && req.OutputPath != req.ManagedReportPath { + if req.OutputPath != "" && req.GenerationErr == nil { + if req.OutputPath != req.ManagedReportPath { if err := fileutil.CopyFileAtomic(req.ManagedReportPath, req.OutputPath); err != nil { return result, err } + result.OutputPath = req.OutputPath + req.ExecutionArtifact.Paths.OutputPath = req.OutputPath + if _, err := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); err != nil { + return result, err + } + } else { + result.OutputPath = req.OutputPath } - result.OutputPath = req.OutputPath } metadata := req.Metadata @@ -588,6 +598,10 @@ func finalizeRenderedReport(ctx context.Context, req finalizeRenderedReportReque result.Notification = notification metadata.NotificationPath = notificationPath result.Metadata = metadata + req.ExecutionArtifact.Paths.NotificationPath = notificationPath + if _, saveErr := req.Store.SavePromptExecution(ctx, req.Resolved, *req.ExecutionArtifact); saveErr != nil { + return result, saveErr + } metadataPath, saveErr := req.Store.SaveMetadata(ctx, metadata) if saveErr != nil { return result, saveErr diff --git a/internal/app/prompt_artifact_paths_test.go b/internal/app/prompt_artifact_paths_test.go index 468c9ee..3c63208 100644 --- a/internal/app/prompt_artifact_paths_test.go +++ b/internal/app/prompt_artifact_paths_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" "time" @@ -18,24 +19,60 @@ import ( ) const ( - failPromptExecution = "prompt execution" - failMetadata = "metadata" + failPromptExecution = "prompt execution" + failMetadata = "metadata" + failGeneratedText = "generated text" + failRenderContext = "render context" + failRenderedReportPath = "rendered report path" + failDistributorNotification = "distributor notification" ) type failingPersistenceStore struct { state.Store - failOperation string - failMetadataCall int - metadataCalls int + failOperation string + failExecutionCall int + failMetadataCall int + executionCalls int + metadataCalls int + renderedReportPath string } func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) { - if s.failOperation == failPromptExecution { + s.executionCalls++ + if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) { return "", errors.New("injected prompt execution persistence failure") } return s.Store.SavePromptExecution(ctx, resolved, artifact) } +func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) { + if s.failOperation == failGeneratedText { + return "", errors.New("injected generated text persistence failure") + } + return s.Store.SaveGeneratedText(ctx, resolved, data) +} + +func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) { + if s.failOperation == failRenderContext { + return "", errors.New("injected render context persistence failure") + } + return s.Store.SaveRenderContext(ctx, resolved, value) +} + +func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) { + if s.failOperation == failRenderedReportPath { + return s.renderedReportPath, nil + } + return s.Store.PrepareRenderedReport(ctx, resolved) +} + +func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) { + if s.failOperation == failDistributorNotification { + return "", errors.New("injected notification persistence failure") + } + return s.Store.SaveDistributorNotification(ctx, resolved, artifact) +} + func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) { s.metadataCalls++ if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall { @@ -93,6 +130,12 @@ func (successfulNotifier) Notify(context.Context, NotificationRequest) (*Notific return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil } +type failingNotifier struct{} + +func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) { + return nil, errors.New("injected notification failure") +} + func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { tests := []struct { name string @@ -123,6 +166,7 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { } if test.notify { req.Config.Notify.Distributor.Enabled = true + req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" req.Notifier = successfulNotifier{} req.noNotify = false } @@ -138,9 +182,12 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) { func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) { tests := []struct { - name string - executor artifactPathExecutor - want reachedPromptArtifacts + name string + executor artifactPathExecutor + want reachedPromptArtifacts + wantExecutionStatus state.PromptExecutionStatus + wantExecutionPaths state.PromptExecutionPaths + wantRawExecution bool }{ { name: "preparation failure", @@ -148,29 +195,262 @@ func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) { want: reachedPromptArtifacts{preparation: true, metadata: true}, }, { - name: "operational execution failure", - executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)}, - want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true}, + name: "operational execution failure", + executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)}, + want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true}, + wantExecutionStatus: state.PromptExecutionFailed, }, { - name: "completed validation rejection", - executor: artifactPathExecutor{validation: promptexec.ValidationFailed}, - want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, + name: "completed validation rejection", + executor: artifactPathExecutor{validation: promptexec.ValidationFailed}, + want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, + wantExecutionStatus: state.PromptExecutionValidationRejected, + wantRawExecution: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { req, paths := promptArtifactRequest(t, test.executor) + if test.wantRawExecution { + test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw + } result, err := generatePromptReport(context.Background(), req) if err == nil || result == nil { t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) } assertReachedPromptArtifacts(t, result, paths, test.want) + if test.wantExecutionStatus != "" { + artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath) + if loadErr != nil { + t.Fatalf("LoadPromptExecution() error = %v", loadErr) + } + if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths { + t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths) + } + } }) } } +func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) { + req, paths := promptArtifactRequest(t, artifactPathExecutor{}) + req.OutputPath = filepath.Join(t.TempDir(), "daily.md") + paths.output = req.OutputPath + req.Config.Notify.Distributor.Enabled = true + req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" + req.Notifier = successfulNotifier{} + req.noNotify = false + + result, err := generatePromptReport(context.Background(), req) + if err != nil { + t.Fatalf("generatePromptReport() error = %v", err) + } + want := state.PromptExecutionPaths{ + RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText, + RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport, + OutputPath: paths.output, NotificationPath: paths.Notification, + } + assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want) + + data, err := os.ReadFile(result.ExecutionPath) + if err != nil { + t.Fatalf("read execution artifact: %v", err) + } + text := string(data) + for _, forbidden := range []string{ + "Showers are possible during the selected day", `"rawOutput":`, `"debug":`, + `"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`, + "credential", "secret-value", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text) + } + } +} + +func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) { + tests := []struct { + name string + failOperation string + failExecutionCall int + failMetadataCall int + requestOutput bool + failOutputCopy bool + notify bool + notificationFailure bool + wantExecution reachedExecutionArtifacts + wantResult reachedPromptArtifacts + }{ + { + name: "normalized text write", failOperation: failGeneratedText, + wantExecution: reachedExecutionArtifacts{raw: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}, + }, + { + name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2, + wantExecution: reachedExecutionArtifacts{raw: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, + }, + { + name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, + }, + { + name: "render context write", failOperation: failRenderContext, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}, + }, + { + name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, + }, + { + name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, + }, + { + name: "managed report write", failOperation: failRenderedReportPath, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}, + }, + { + name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, + }, + { + name: "output copy write", requestOutput: true, failOutputCopy: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}, + }, + { + name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, + }, + { + name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, + }, + { + name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}, + }, + { + name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, + }, + { + name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, + }, + { + name: "notification operation", requestOutput: true, notify: true, notificationFailure: true, + wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, + wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, paths := promptArtifactRequest(t, artifactPathExecutor{}) + store := &failingPersistenceStore{ + Store: req.Store, failOperation: test.failOperation, + failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall, + } + if test.failOperation == failRenderedReportPath { + store.renderedReportPath = t.TempDir() + } + req.Store = store + if test.requestOutput { + req.OutputPath = filepath.Join(t.TempDir(), "daily.md") + paths.output = req.OutputPath + } + if test.failOutputCopy { + blocker := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil { + t.Fatalf("write output blocker: %v", err) + } + req.OutputPath = filepath.Join(blocker, "daily.md") + paths.output = req.OutputPath + } + if test.notify { + req.Config.Notify.Distributor.Enabled = true + req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter" + req.Notifier = successfulNotifier{} + req.noNotify = false + } + if test.notificationFailure { + req.Notifier = failingNotifier{} + } + + result, err := generatePromptReport(context.Background(), req) + if err == nil || result == nil { + t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err) + } + assertReachedPromptArtifacts(t, result, paths, test.wantResult) + assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution)) + }) + } +} + +type reachedExecutionArtifacts struct { + raw bool + normalized bool + renderContext bool + report bool + output bool + notification bool +} + +func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths { + result := state.PromptExecutionPaths{} + if reached.raw { + result.RawOutputPath = paths.GeneratedTextRaw + } + if reached.normalized { + result.GeneratedTextPath = paths.GeneratedText + } + if reached.renderContext { + result.RenderContextPath = paths.RenderContext + } + if reached.report { + result.RenderedReportPath = paths.RenderedReport + } + if reached.output { + result.OutputPath = paths.output + } + if reached.notification { + result.NotificationPath = paths.Notification + } + return result +} + +func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) { + t.Helper() + artifact, err := store.LoadPromptExecution(context.Background(), path) + if err != nil { + t.Fatalf("LoadPromptExecution() error = %v", err) + } + if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed { + t.Fatalf("execution outcome changed after downstream write: %#v", artifact) + } + if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" { + t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance) + } + if artifact.Paths != want { + t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want) + } +} + type promptArtifactPaths struct { state.ArtifactPaths output string diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index 3e87c8c..8cacedb 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -287,6 +287,10 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report metadata.GeneratedTextPath = generatedTextPath result.GeneratedTextPath = generatedTextPath result.Metadata = metadata + executionArtifact.Paths.GeneratedTextPath = generatedTextPath + if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil { + return result, err + } metadataPath, err = store.SaveMetadata(ctx, metadata) if err != nil { return result, err @@ -304,6 +308,10 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report metadata.RenderContextPath = renderContextPath result.RenderContextPath = renderContextPath result.Metadata = metadata + executionArtifact.Paths.RenderContextPath = renderContextPath + if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil { + return result, err + } metadataPath, err = store.SaveMetadata(ctx, metadata) if err != nil { return result, err @@ -324,9 +332,14 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report result.ReportPath = reportPath metadata.RenderedReportPath = reportPath result.Metadata = metadata + executionArtifact.Paths.RenderedReportPath = reportPath + if _, err := store.SavePromptExecution(ctx, req.Resolved, executionArtifact); err != nil { + return result, err + } finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{ Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata, MetadataPath: result.MetadataPath, - ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier, noNotify: req.noNotify, + ExecutionArtifact: &executionArtifact, ManagedReportPath: reportPath, OutputPath: req.OutputPath, + Notifier: req.Notifier, noNotify: req.noNotify, }) result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath result.Metadata, result.MetadataPath, result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification