eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Track completed execution artifact paths

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-31 16:51:12 +00:00
parent 870b54a4a0
commit 34c395d7e5
5 changed files with 337 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
internal/app/prompt_artifact_paths_test.go
View File

@@ -6,6 +6,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -18,24 +19,60 @@ import (
)
const (
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failPromptExecution = "prompt execution"
failMetadata = "metadata"
failGeneratedText = "generated text"
failRenderContext = "render context"
failRenderedReportPath = "rendered report path"
failDistributorNotification = "distributor notification"
)
type failingPersistenceStore struct {
state.Store
failOperation string
failMetadataCall int
metadataCalls int
failOperation string
failExecutionCall int
failMetadataCall int
executionCalls int
metadataCalls int
renderedReportPath string
}
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
if s.failOperation == failPromptExecution {
s.executionCalls++
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
return "", errors.New("injected prompt execution persistence failure")
}
return s.Store.SavePromptExecution(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
if s.failOperation == failGeneratedText {
return "", errors.New("injected generated text persistence failure")
}
return s.Store.SaveGeneratedText(ctx, resolved, data)
}
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
if s.failOperation == failRenderContext {
return "", errors.New("injected render context persistence failure")
}
return s.Store.SaveRenderContext(ctx, resolved, value)
}
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
if s.failOperation == failRenderedReportPath {
return s.renderedReportPath, nil
}
return s.Store.PrepareRenderedReport(ctx, resolved)
}
func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) {
if s.failOperation == failDistributorNotification {
return "", errors.New("injected notification persistence failure")
}
return s.Store.SaveDistributorNotification(ctx, resolved, artifact)
}
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
s.metadataCalls++
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
@@ -93,6 +130,12 @@ func (successfulNotifier) Notify(context.Context, NotificationRequest) (*Notific
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
}
type failingNotifier struct{}
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
return nil, errors.New("injected notification failure")
}
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
tests := []struct {
name string
@@ -123,6 +166,7 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
@@ -138,9 +182,12 @@ func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
tests := []struct {
name string
executor artifactPathExecutor
want reachedPromptArtifacts
name string
executor artifactPathExecutor
want reachedPromptArtifacts
wantExecutionStatus state.PromptExecutionStatus
wantExecutionPaths state.PromptExecutionPaths
wantRawExecution bool
}{
{
name: "preparation failure",
@@ -148,29 +195,262 @@ func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
want: reachedPromptArtifacts{preparation: true, metadata: true},
},
{
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
name: "operational execution failure",
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
wantExecutionStatus: state.PromptExecutionFailed,
},
{
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
name: "completed validation rejection",
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
wantExecutionStatus: state.PromptExecutionValidationRejected,
wantRawExecution: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, test.executor)
if test.wantRawExecution {
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.want)
if test.wantExecutionStatus != "" {
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if loadErr != nil {
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
}
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
}
}
})
}
}
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
result, err := generatePromptReport(context.Background(), req)
if err != nil {
t.Fatalf("generatePromptReport() error = %v", err)
}
want := state.PromptExecutionPaths{
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
OutputPath: paths.output, NotificationPath: paths.Notification,
}
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
data, err := os.ReadFile(result.ExecutionPath)
if err != nil {
t.Fatalf("read execution artifact: %v", err)
}
text := string(data)
for _, forbidden := range []string{
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
"credential", "secret-value",
} {
if strings.Contains(text, forbidden) {
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
}
}
}
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
tests := []struct {
name string
failOperation string
failExecutionCall int
failMetadataCall int
requestOutput bool
failOutputCopy bool
notify bool
notificationFailure bool
wantExecution reachedExecutionArtifacts
wantResult reachedPromptArtifacts
}{
{
name: "normalized text write", failOperation: failGeneratedText,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
},
{
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
wantExecution: reachedExecutionArtifacts{raw: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context write", failOperation: failRenderContext,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
},
{
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report write", failOperation: failRenderedReportPath,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
},
{
name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy write", requestOutput: true, failOutputCopy: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
},
{
name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
},
{
name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
{
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
store := &failingPersistenceStore{
Store: req.Store, failOperation: test.failOperation,
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
}
if test.failOperation == failRenderedReportPath {
store.renderedReportPath = t.TempDir()
}
req.Store = store
if test.requestOutput {
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
paths.output = req.OutputPath
}
if test.failOutputCopy {
blocker := filepath.Join(t.TempDir(), "not-a-directory")
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
t.Fatalf("write output blocker: %v", err)
}
req.OutputPath = filepath.Join(blocker, "daily.md")
paths.output = req.OutputPath
}
if test.notify {
req.Config.Notify.Distributor.Enabled = true
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
req.Notifier = successfulNotifier{}
req.noNotify = false
}
if test.notificationFailure {
req.Notifier = failingNotifier{}
}
result, err := generatePromptReport(context.Background(), req)
if err == nil || result == nil {
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
}
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
})
}
}
type reachedExecutionArtifacts struct {
raw bool
normalized bool
renderContext bool
report bool
output bool
notification bool
}
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
result := state.PromptExecutionPaths{}
if reached.raw {
result.RawOutputPath = paths.GeneratedTextRaw
}
if reached.normalized {
result.GeneratedTextPath = paths.GeneratedText
}
if reached.renderContext {
result.RenderContextPath = paths.RenderContext
}
if reached.report {
result.RenderedReportPath = paths.RenderedReport
}
if reached.output {
result.OutputPath = paths.output
}
if reached.notification {
result.NotificationPath = paths.Notification
}
return result
}
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
t.Helper()
artifact, err := store.LoadPromptExecution(context.Background(), path)
if err != nil {
t.Fatalf("LoadPromptExecution() error = %v", err)
}
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
}
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
}
if artifact.Paths != want {
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
}
}
type promptArtifactPaths struct {
state.ArtifactPaths
output string