553 lines
25 KiB
Go
553 lines
25 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
const (
|
|
failPromptExecution = "prompt execution"
|
|
failMetadata = "metadata"
|
|
failGeneratedText = "generated text"
|
|
failRenderContext = "render context"
|
|
failRenderedReportPath = "rendered report path"
|
|
failDistributorNotification = "distributor notification"
|
|
)
|
|
|
|
type failingPersistenceStore struct {
|
|
state.Store
|
|
failOperation string
|
|
failExecutionCall int
|
|
failMetadataCall int
|
|
executionCalls int
|
|
metadataCalls int
|
|
renderedReportPath string
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
|
|
s.executionCalls++
|
|
if s.failOperation == failPromptExecution && (s.failExecutionCall == 0 || s.executionCalls == s.failExecutionCall) {
|
|
return "", errors.New("injected prompt execution persistence failure")
|
|
}
|
|
return s.Store.SavePromptExecution(ctx, resolved, artifact)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SaveGeneratedText(ctx context.Context, resolved report.Resolved, data []byte) (string, error) {
|
|
if s.failOperation == failGeneratedText {
|
|
return "", errors.New("injected generated text persistence failure")
|
|
}
|
|
return s.Store.SaveGeneratedText(ctx, resolved, data)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SaveRenderContext(ctx context.Context, resolved report.Resolved, value any) (string, error) {
|
|
if s.failOperation == failRenderContext {
|
|
return "", errors.New("injected render context persistence failure")
|
|
}
|
|
return s.Store.SaveRenderContext(ctx, resolved, value)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) PrepareRenderedReport(ctx context.Context, resolved report.Resolved) (string, error) {
|
|
if s.failOperation == failRenderedReportPath {
|
|
return s.renderedReportPath, nil
|
|
}
|
|
return s.Store.PrepareRenderedReport(ctx, resolved)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SaveDistributorNotification(ctx context.Context, resolved report.Resolved, artifact state.DistributorNotificationArtifact) (string, error) {
|
|
if s.failOperation == failDistributorNotification {
|
|
return "", errors.New("injected notification persistence failure")
|
|
}
|
|
return s.Store.SaveDistributorNotification(ctx, resolved, artifact)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
|
|
s.metadataCalls++
|
|
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
|
|
return "", errors.New("injected metadata persistence failure")
|
|
}
|
|
return s.Store.SaveMetadata(ctx, metadata)
|
|
}
|
|
|
|
type artifactPathExecutor struct {
|
|
beforePreparationErr error
|
|
afterPreparationErr error
|
|
validation promptexec.ValidationStatus
|
|
}
|
|
|
|
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
|
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
|
|
}
|
|
|
|
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
|
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
|
|
}
|
|
|
|
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
if e.beforePreparationErr != nil {
|
|
return nil, e.beforePreparationErr
|
|
}
|
|
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
if err := callback(promptexec.Preparation{
|
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
|
|
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
|
|
}, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
if e.afterPreparationErr != nil {
|
|
return nil, e.afterPreparationErr
|
|
}
|
|
validation := e.validation
|
|
if validation == "" {
|
|
validation = promptexec.ValidationPassed
|
|
}
|
|
return &promptexec.Execution{
|
|
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
|
|
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
|
|
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`),
|
|
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
|
|
}, nil
|
|
}
|
|
|
|
type successfulNotifier struct{}
|
|
|
|
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
|
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
|
|
}
|
|
|
|
type failingNotifier struct{}
|
|
|
|
func (failingNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
|
return nil, errors.New("injected notification failure")
|
|
}
|
|
|
|
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
failOperation string
|
|
failMetadataCall int
|
|
outputCopy bool
|
|
notify bool
|
|
want reachedPromptArtifacts
|
|
}{
|
|
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
|
|
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
|
|
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
|
|
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
|
|
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
|
|
{name: "managed report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
|
|
{name: "output copy then metadata", failOperation: failMetadata, failMetadataCall: 5, outputCopy: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
|
|
{name: "notification then metadata", failOperation: failMetadata, failMetadataCall: 6, notify: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, notification: true}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
|
|
req.Store = store
|
|
if test.outputCopy {
|
|
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
paths.output = req.OutputPath
|
|
}
|
|
if test.notify {
|
|
req.Config.Notify.Distributor.Enabled = true
|
|
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
req.Notifier = successfulNotifier{}
|
|
req.noNotify = false
|
|
}
|
|
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
}
|
|
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
executor artifactPathExecutor
|
|
want reachedPromptArtifacts
|
|
wantExecutionStatus state.PromptExecutionStatus
|
|
wantExecutionPaths state.PromptExecutionPaths
|
|
wantRawExecution bool
|
|
}{
|
|
{
|
|
name: "preparation failure",
|
|
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
|
|
want: reachedPromptArtifacts{preparation: true, metadata: true},
|
|
},
|
|
{
|
|
name: "operational execution failure",
|
|
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
|
|
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
|
|
wantExecutionStatus: state.PromptExecutionFailed,
|
|
},
|
|
{
|
|
name: "completed validation rejection",
|
|
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
|
|
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
|
wantExecutionStatus: state.PromptExecutionValidationRejected,
|
|
wantRawExecution: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, test.executor)
|
|
if test.wantRawExecution {
|
|
test.wantExecutionPaths.RawOutputPath = paths.GeneratedTextRaw
|
|
}
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
}
|
|
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
if test.wantExecutionStatus != "" {
|
|
artifact, loadErr := req.Store.LoadPromptExecution(context.Background(), result.ExecutionPath)
|
|
if loadErr != nil {
|
|
t.Fatalf("LoadPromptExecution() error = %v", loadErr)
|
|
}
|
|
if artifact.Status != test.wantExecutionStatus || artifact.Paths != test.wantExecutionPaths {
|
|
t.Fatalf("execution outcome/paths = %q/%#v, want %q/%#v", artifact.Status, artifact.Paths, test.wantExecutionStatus, test.wantExecutionPaths)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompletedExecutionArtifactTracksDownstreamLifecycle(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
paths.output = req.OutputPath
|
|
req.Config.Notify.Distributor.Enabled = true
|
|
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
req.Notifier = successfulNotifier{}
|
|
req.noNotify = false
|
|
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("generatePromptReport() error = %v", err)
|
|
}
|
|
want := state.PromptExecutionPaths{
|
|
RawOutputPath: paths.GeneratedTextRaw, GeneratedTextPath: paths.GeneratedText,
|
|
RenderContextPath: paths.RenderContext, RenderedReportPath: paths.RenderedReport,
|
|
OutputPath: paths.output, NotificationPath: paths.Notification,
|
|
}
|
|
assertPersistedExecutionPaths(t, req.Store, result.ExecutionPath, want)
|
|
|
|
data, err := os.ReadFile(result.ExecutionPath)
|
|
if err != nil {
|
|
t.Fatalf("read execution artifact: %v", err)
|
|
}
|
|
text := string(data)
|
|
for _, forbidden := range []string{
|
|
"Showers are possible during the selected day", `"rawOutput":`, `"debug":`,
|
|
`"renderedMessages":`, `"structuredSchema":`, `"endpoint":`, `"parametersJSON":`,
|
|
"credential", "secret-value",
|
|
} {
|
|
if strings.Contains(text, forbidden) {
|
|
t.Fatalf("execution artifact contains unsafe generated or provider detail %q:\n%s", forbidden, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCompletedExecutionArtifactRetainsLastPersistedCheckpoint(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
failOperation string
|
|
failExecutionCall int
|
|
failMetadataCall int
|
|
requestOutput bool
|
|
failOutputCopy bool
|
|
notify bool
|
|
notificationFailure bool
|
|
wantExecution reachedExecutionArtifacts
|
|
wantResult reachedPromptArtifacts
|
|
}{
|
|
{
|
|
name: "normalized text write", failOperation: failGeneratedText,
|
|
wantExecution: reachedExecutionArtifacts{raw: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
|
},
|
|
{
|
|
name: "normalized text checkpoint", failOperation: failPromptExecution, failExecutionCall: 2,
|
|
wantExecution: reachedExecutionArtifacts{raw: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
},
|
|
{
|
|
name: "normalized text metadata", failOperation: failMetadata, failMetadataCall: 3,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
},
|
|
{
|
|
name: "render context write", failOperation: failRenderContext,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true},
|
|
},
|
|
{
|
|
name: "render context checkpoint", failOperation: failPromptExecution, failExecutionCall: 3,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
},
|
|
{
|
|
name: "render context metadata", failOperation: failMetadata, failMetadataCall: 4,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
},
|
|
{
|
|
name: "managed report write", failOperation: failRenderedReportPath,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true},
|
|
},
|
|
{
|
|
name: "managed report checkpoint", failOperation: failPromptExecution, failExecutionCall: 4,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
|
},
|
|
{
|
|
name: "output copy write", requestOutput: true, failOutputCopy: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true},
|
|
},
|
|
{
|
|
name: "output copy checkpoint", failOperation: failPromptExecution, failExecutionCall: 5, requestOutput: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
},
|
|
{
|
|
name: "output copy metadata", failOperation: failMetadata, failMetadataCall: 5, requestOutput: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
},
|
|
{
|
|
name: "notification artifact write", failOperation: failDistributorNotification, requestOutput: true, notify: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
},
|
|
{
|
|
name: "notification checkpoint", failOperation: failPromptExecution, failExecutionCall: 6, requestOutput: true, notify: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
|
|
},
|
|
{
|
|
name: "notification metadata", failOperation: failMetadata, failMetadataCall: 6, requestOutput: true, notify: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
|
|
},
|
|
{
|
|
name: "notification operation", requestOutput: true, notify: true, notificationFailure: true,
|
|
wantExecution: reachedExecutionArtifacts{raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
|
|
wantResult: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true, notification: true},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
store := &failingPersistenceStore{
|
|
Store: req.Store, failOperation: test.failOperation,
|
|
failExecutionCall: test.failExecutionCall, failMetadataCall: test.failMetadataCall,
|
|
}
|
|
if test.failOperation == failRenderedReportPath {
|
|
store.renderedReportPath = t.TempDir()
|
|
}
|
|
req.Store = store
|
|
if test.requestOutput {
|
|
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
paths.output = req.OutputPath
|
|
}
|
|
if test.failOutputCopy {
|
|
blocker := filepath.Join(t.TempDir(), "not-a-directory")
|
|
if err := os.WriteFile(blocker, []byte("block"), 0o600); err != nil {
|
|
t.Fatalf("write output blocker: %v", err)
|
|
}
|
|
req.OutputPath = filepath.Join(blocker, "daily.md")
|
|
paths.output = req.OutputPath
|
|
}
|
|
if test.notify {
|
|
req.Config.Notify.Distributor.Enabled = true
|
|
req.Config.Notify.Distributor.PipelineIDTemplate = "weatherreporter"
|
|
req.Notifier = successfulNotifier{}
|
|
req.noNotify = false
|
|
}
|
|
if test.notificationFailure {
|
|
req.Notifier = failingNotifier{}
|
|
}
|
|
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
}
|
|
assertReachedPromptArtifacts(t, result, paths, test.wantResult)
|
|
assertPersistedExecutionPaths(t, store, result.ExecutionPath, executionPathsFor(paths, test.wantExecution))
|
|
})
|
|
}
|
|
}
|
|
|
|
type reachedExecutionArtifacts struct {
|
|
raw bool
|
|
normalized bool
|
|
renderContext bool
|
|
report bool
|
|
output bool
|
|
notification bool
|
|
}
|
|
|
|
func executionPathsFor(paths promptArtifactPaths, reached reachedExecutionArtifacts) state.PromptExecutionPaths {
|
|
result := state.PromptExecutionPaths{}
|
|
if reached.raw {
|
|
result.RawOutputPath = paths.GeneratedTextRaw
|
|
}
|
|
if reached.normalized {
|
|
result.GeneratedTextPath = paths.GeneratedText
|
|
}
|
|
if reached.renderContext {
|
|
result.RenderContextPath = paths.RenderContext
|
|
}
|
|
if reached.report {
|
|
result.RenderedReportPath = paths.RenderedReport
|
|
}
|
|
if reached.output {
|
|
result.OutputPath = paths.output
|
|
}
|
|
if reached.notification {
|
|
result.NotificationPath = paths.Notification
|
|
}
|
|
return result
|
|
}
|
|
|
|
func assertPersistedExecutionPaths(t *testing.T, store state.Store, path string, want state.PromptExecutionPaths) {
|
|
t.Helper()
|
|
artifact, err := store.LoadPromptExecution(context.Background(), path)
|
|
if err != nil {
|
|
t.Fatalf("LoadPromptExecution() error = %v", err)
|
|
}
|
|
if artifact.Status != state.PromptExecutionSucceeded || artifact.Validation == nil || artifact.Validation.Status != promptexec.ValidationPassed {
|
|
t.Fatalf("execution outcome changed after downstream write: %#v", artifact)
|
|
}
|
|
if artifact.Provenance == nil || artifact.Provenance.RunID != "provider-run" || artifact.Provenance.PromptHash != "prompt-hash" {
|
|
t.Fatalf("execution provenance changed after downstream write: %#v", artifact.Provenance)
|
|
}
|
|
if artifact.Paths != want {
|
|
t.Fatalf("execution paths = %#v, want %#v", artifact.Paths, want)
|
|
}
|
|
}
|
|
|
|
type promptArtifactPaths struct {
|
|
state.ArtifactPaths
|
|
output string
|
|
}
|
|
|
|
type reachedPromptArtifacts struct {
|
|
preparation bool
|
|
execution bool
|
|
metadata bool
|
|
raw bool
|
|
normalized bool
|
|
renderContext bool
|
|
report bool
|
|
output bool
|
|
notification bool
|
|
}
|
|
|
|
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
|
|
t.Helper()
|
|
cfg := config.Defaults()
|
|
cfg.Workspace.Root = t.TempDir()
|
|
resolved, err := ResolveGenerate(GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
|
|
}, mustParse("2026-05-29T05:00:00-05:00"))
|
|
if err != nil {
|
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
}
|
|
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
if err != nil {
|
|
t.Fatalf("read daily fixture: %v", err)
|
|
}
|
|
var bundle weatherdata.Bundle
|
|
if err := json.Unmarshal(bundleData, &bundle); err != nil {
|
|
t.Fatalf("decode daily fixture: %v", err)
|
|
}
|
|
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
|
|
if err != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
}
|
|
paths, err := filesystemStore.Paths(resolved)
|
|
if err != nil {
|
|
t.Fatalf("Paths() error = %v", err)
|
|
}
|
|
debugWriter, err := state.NewPromptDebugWriter("")
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
return promptReportRequest{
|
|
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
|
|
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
|
|
Inspection: PromptInspectionResult{
|
|
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
|
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
|
|
},
|
|
DebugWriter: debugWriter, noNotify: true,
|
|
}, promptArtifactPaths{ArtifactPaths: paths}
|
|
}
|
|
|
|
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
|
|
t.Helper()
|
|
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
|
|
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
|
|
}
|
|
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
|
|
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
|
|
}
|
|
checks := []struct {
|
|
name string
|
|
got string
|
|
metadataGot string
|
|
inMetadata bool
|
|
path string
|
|
want bool
|
|
}{
|
|
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
|
|
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
|
|
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
|
|
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
|
|
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
|
|
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
|
|
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
|
|
{"output", result.OutputPath, "", false, paths.output, want.output},
|
|
{"notification", result.NotificationPath, result.Metadata.NotificationPath, true, paths.Notification, want.notification},
|
|
}
|
|
for _, check := range checks {
|
|
if check.want && check.got != check.path {
|
|
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
|
|
}
|
|
if check.want && check.inMetadata && check.metadataGot != check.path {
|
|
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
|
|
}
|
|
if !check.want && check.got != "" {
|
|
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
|
|
}
|
|
if !check.want && check.inMetadata && check.metadataGot != "" {
|
|
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
|
|
}
|
|
}
|
|
}
|