273 lines
12 KiB
Go
273 lines
12 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/state"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
const (
|
|
failPromptExecution = "prompt execution"
|
|
failMetadata = "metadata"
|
|
)
|
|
|
|
type failingPersistenceStore struct {
|
|
state.Store
|
|
failOperation string
|
|
failMetadataCall int
|
|
metadataCalls int
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SavePromptExecution(ctx context.Context, resolved report.Resolved, artifact state.PromptExecutionArtifact) (string, error) {
|
|
if s.failOperation == failPromptExecution {
|
|
return "", errors.New("injected prompt execution persistence failure")
|
|
}
|
|
return s.Store.SavePromptExecution(ctx, resolved, artifact)
|
|
}
|
|
|
|
func (s *failingPersistenceStore) SaveMetadata(ctx context.Context, metadata state.Metadata) (string, error) {
|
|
s.metadataCalls++
|
|
if s.failOperation == failMetadata && s.metadataCalls == s.failMetadataCall {
|
|
return "", errors.New("injected metadata persistence failure")
|
|
}
|
|
return s.Store.SaveMetadata(ctx, metadata)
|
|
}
|
|
|
|
type artifactPathExecutor struct {
|
|
beforePreparationErr error
|
|
afterPreparationErr error
|
|
validation promptexec.ValidationStatus
|
|
}
|
|
|
|
func (e artifactPathExecutor) InspectPrompt(context.Context, string, string) (promptexec.PromptInspection, error) {
|
|
return promptexec.PromptInspection{}, errors.New("unexpected inspection")
|
|
}
|
|
|
|
func (e artifactPathExecutor) InspectProfile(context.Context, string) (promptexec.ProfileInspection, error) {
|
|
return promptexec.ProfileInspection{}, errors.New("unexpected inspection")
|
|
}
|
|
|
|
func (e artifactPathExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
if e.beforePreparationErr != nil {
|
|
return nil, e.beforePreparationErr
|
|
}
|
|
now := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
if err := callback(promptexec.Preparation{
|
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
|
|
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "test",
|
|
ModelName: "test-model", DataPackagePath: req.DataPackagePath, StartedAt: now, EndedAt: now,
|
|
}, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
if e.afterPreparationErr != nil {
|
|
return nil, e.afterPreparationErr
|
|
}
|
|
validation := e.validation
|
|
if validation == "" {
|
|
validation = promptexec.ValidationPassed
|
|
}
|
|
return &promptexec.Execution{
|
|
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
|
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
|
|
BackendID: "test", ModelName: "test-model", GeneratedHash: "generated-hash",
|
|
StartedAt: now, EndedAt: now, DataPackagePath: req.DataPackagePath,
|
|
RawOutput: []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon.","confidence":"Medium"}`),
|
|
Validation: promptexec.NewValidation(validation, "json_schema", "daily.generated_text.schema.json", nil),
|
|
}, nil
|
|
}
|
|
|
|
type successfulNotifier struct{}
|
|
|
|
func (successfulNotifier) Notify(context.Context, NotificationRequest) (*NotificationResult, error) {
|
|
return &NotificationResult{RunID: "notification-run", Status: "succeeded", UploadStatus: "accepted"}, nil
|
|
}
|
|
|
|
func TestGeneratePromptReportReturnsOnlyReachedArtifactPaths(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
failOperation string
|
|
failMetadataCall int
|
|
outputCopy bool
|
|
notify bool
|
|
want reachedPromptArtifacts
|
|
}{
|
|
{name: "preparation then metadata", failOperation: failMetadata, failMetadataCall: 1, want: reachedPromptArtifacts{preparation: true}},
|
|
{name: "raw output then execution", failOperation: failPromptExecution, want: reachedPromptArtifacts{preparation: true, metadata: true, raw: true}},
|
|
{name: "execution then metadata", failOperation: failMetadata, failMetadataCall: 2, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true}},
|
|
{name: "normalized output then metadata", failOperation: failMetadata, failMetadataCall: 3, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true}},
|
|
{name: "render context then metadata", failOperation: failMetadata, failMetadataCall: 4, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true}},
|
|
{name: "managed report then metadata", failOperation: failMetadata, failMetadataCall: 5, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true}},
|
|
{name: "output copy then metadata", failOperation: failMetadata, failMetadataCall: 5, outputCopy: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, output: true}},
|
|
{name: "notification then metadata", failOperation: failMetadata, failMetadataCall: 6, notify: true, want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true, normalized: true, renderContext: true, report: true, notification: true}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, artifactPathExecutor{})
|
|
store := &failingPersistenceStore{Store: req.Store, failOperation: test.failOperation, failMetadataCall: test.failMetadataCall}
|
|
req.Store = store
|
|
if test.outputCopy {
|
|
req.OutputPath = filepath.Join(t.TempDir(), "daily.md")
|
|
paths.output = req.OutputPath
|
|
}
|
|
if test.notify {
|
|
req.Config.Notify.Distributor.Enabled = true
|
|
req.Notifier = successfulNotifier{}
|
|
req.noNotify = false
|
|
}
|
|
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
}
|
|
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGeneratePromptReportFailureReceiptsExposeReachedPaths(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
executor artifactPathExecutor
|
|
want reachedPromptArtifacts
|
|
}{
|
|
{
|
|
name: "preparation failure",
|
|
executor: artifactPathExecutor{beforePreparationErr: promptexec.NewError(promptexec.Generation, "prepare failed", nil)},
|
|
want: reachedPromptArtifacts{preparation: true, metadata: true},
|
|
},
|
|
{
|
|
name: "operational execution failure",
|
|
executor: artifactPathExecutor{afterPreparationErr: promptexec.NewError(promptexec.Generation, "provider failed", nil)},
|
|
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true},
|
|
},
|
|
{
|
|
name: "completed validation rejection",
|
|
executor: artifactPathExecutor{validation: promptexec.ValidationFailed},
|
|
want: reachedPromptArtifacts{preparation: true, execution: true, metadata: true, raw: true},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
req, paths := promptArtifactRequest(t, test.executor)
|
|
result, err := generatePromptReport(context.Background(), req)
|
|
if err == nil || result == nil {
|
|
t.Fatalf("generatePromptReport() result/error = %#v/%v, want partial result and failure", result, err)
|
|
}
|
|
assertReachedPromptArtifacts(t, result, paths, test.want)
|
|
})
|
|
}
|
|
}
|
|
|
|
type promptArtifactPaths struct {
|
|
state.ArtifactPaths
|
|
output string
|
|
}
|
|
|
|
type reachedPromptArtifacts struct {
|
|
preparation bool
|
|
execution bool
|
|
metadata bool
|
|
raw bool
|
|
normalized bool
|
|
renderContext bool
|
|
report bool
|
|
output bool
|
|
notification bool
|
|
}
|
|
|
|
func promptArtifactRequest(t *testing.T, executor promptexec.Executor) (promptReportRequest, promptArtifactPaths) {
|
|
t.Helper()
|
|
cfg := config.Defaults()
|
|
cfg.Workspace.Root = t.TempDir()
|
|
resolved, err := ResolveGenerate(GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: mustParse("2026-05-29T12:00:00-05:00"),
|
|
}, mustParse("2026-05-29T05:00:00-05:00"))
|
|
if err != nil {
|
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
}
|
|
bundleData, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
if err != nil {
|
|
t.Fatalf("read daily fixture: %v", err)
|
|
}
|
|
var bundle weatherdata.Bundle
|
|
if err := json.Unmarshal(bundleData, &bundle); err != nil {
|
|
t.Fatalf("decode daily fixture: %v", err)
|
|
}
|
|
filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
|
|
if err != nil {
|
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
|
}
|
|
paths, err := filesystemStore.Paths(resolved)
|
|
if err != nil {
|
|
t.Fatalf("Paths() error = %v", err)
|
|
}
|
|
debugWriter, err := state.NewPromptDebugWriter("")
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
return promptReportRequest{
|
|
GenerateRequest: GenerateRequest{Config: cfg, Report: ReportDaily, Executor: executor, Store: filesystemStore},
|
|
Resolved: resolved, Collection: collect.Result{Bundle: &bundle},
|
|
Inspection: PromptInspectionResult{
|
|
PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion,
|
|
PromptHash: "prompt-hash", ProfileID: "test-profile", BackendID: "test", ModelName: "test-model",
|
|
},
|
|
DebugWriter: debugWriter, noNotify: true,
|
|
}, promptArtifactPaths{ArtifactPaths: paths}
|
|
}
|
|
|
|
func assertReachedPromptArtifacts(t *testing.T, result *ReportResult, paths promptArtifactPaths, want reachedPromptArtifacts) {
|
|
t.Helper()
|
|
if result.ModuleSnapshotPath != paths.ModuleSnapshot || result.DataPackagePath != paths.DataPackage {
|
|
t.Fatalf("base paths = module %q data %q, want %q and %q", result.ModuleSnapshotPath, result.DataPackagePath, paths.ModuleSnapshot, paths.DataPackage)
|
|
}
|
|
if result.Metadata.ModuleSnapshotPath != paths.ModuleSnapshot || result.Metadata.DataPackagePath != paths.DataPackage || result.Metadata.MetadataPath != paths.Metadata {
|
|
t.Fatalf("metadata base paths = %#v, want reached module/data paths and metadata destination", result.Metadata)
|
|
}
|
|
checks := []struct {
|
|
name string
|
|
got string
|
|
metadataGot string
|
|
inMetadata bool
|
|
path string
|
|
want bool
|
|
}{
|
|
{"preparation", result.PreparationPath, result.Metadata.PreparationPath, true, paths.Preparation, want.preparation},
|
|
{"execution", result.ExecutionPath, result.Metadata.ExecutionPath, true, paths.Execution, want.execution},
|
|
{"metadata", result.MetadataPath, "", false, paths.Metadata, want.metadata},
|
|
{"raw", result.GeneratedTextRawPath, result.Metadata.GeneratedTextRawPath, true, paths.GeneratedTextRaw, want.raw},
|
|
{"normalized", result.GeneratedTextPath, result.Metadata.GeneratedTextPath, true, paths.GeneratedText, want.normalized},
|
|
{"render context", result.RenderContextPath, result.Metadata.RenderContextPath, true, paths.RenderContext, want.renderContext},
|
|
{"report", result.ReportPath, result.Metadata.RenderedReportPath, true, paths.RenderedReport, want.report},
|
|
{"output", result.OutputPath, "", false, paths.output, want.output},
|
|
{"notification", result.NotificationPath, result.Metadata.NotificationPath, true, paths.Notification, want.notification},
|
|
}
|
|
for _, check := range checks {
|
|
if check.want && check.got != check.path {
|
|
t.Errorf("%s path = %q, want reached path %q", check.name, check.got, check.path)
|
|
}
|
|
if check.want && check.inMetadata && check.metadataGot != check.path {
|
|
t.Errorf("metadata %s path = %q, want reached path %q", check.name, check.metadataGot, check.path)
|
|
}
|
|
if !check.want && check.got != "" {
|
|
t.Errorf("%s path = %q, want empty because artifact was not reached", check.name, check.got)
|
|
}
|
|
if !check.want && check.inMetadata && check.metadataGot != "" {
|
|
t.Errorf("metadata %s path = %q, want empty because artifact was not reached", check.name, check.metadataGot)
|
|
}
|
|
}
|
|
}
|