251 lines
11 KiB
Go
251 lines
11 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
type generationCollector struct {
|
|
bundle *weatherdata.Bundle
|
|
err error
|
|
called bool
|
|
}
|
|
|
|
func (c *generationCollector) Run(context.Context, collect.Request) (*collect.Result, error) {
|
|
c.called = true
|
|
return &collect.Result{Bundle: c.bundle}, c.err
|
|
}
|
|
|
|
type generationExecutor struct {
|
|
called bool
|
|
inspectErr error
|
|
executeErr error
|
|
respectCancellation bool
|
|
validation promptexec.ValidationStatus
|
|
rawOutput []byte
|
|
failedPrompt string
|
|
}
|
|
|
|
func (e generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
|
if e.inspectErr != nil {
|
|
return promptexec.PromptInspection{}, e.inspectErr
|
|
}
|
|
definition := generationDefinitionForPrompt(id)
|
|
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
|
}
|
|
func (generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
|
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
|
}
|
|
func (e *generationExecutor) Execute(ctx context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
|
if e.respectCancellation && ctx.Err() != nil {
|
|
return nil, ctx.Err()
|
|
}
|
|
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
e.called = true
|
|
if e.executeErr != nil {
|
|
return nil, e.executeErr
|
|
}
|
|
status := e.validation
|
|
if status == "" {
|
|
status = promptexec.ValidationPassed
|
|
}
|
|
if e.failedPrompt == req.PromptID {
|
|
status = promptexec.ValidationFailed
|
|
}
|
|
rawOutput := e.rawOutput
|
|
if rawOutput == nil {
|
|
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
|
}
|
|
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
|
}
|
|
|
|
func generationDefinitionForPrompt(promptID string) report.Definition {
|
|
for _, definition := range report.DefaultRegistry().All() {
|
|
if definition.PromptID == promptID {
|
|
return definition
|
|
}
|
|
}
|
|
panic("unknown fixture prompt " + promptID)
|
|
}
|
|
|
|
func TestGenerateDetailedPublishesOnlySelectedOutput(t *testing.T) {
|
|
cfg := config.Defaults()
|
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
|
bundle := generationBundle(t)
|
|
executor := &generationExecutor{}
|
|
workingDir := t.TempDir()
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: workingDir, Collector: &generationCollector{bundle: &bundle}, Executor: executor})
|
|
if err != nil {
|
|
t.Fatalf("GenerateDetailed() error = %v", err)
|
|
}
|
|
if !executor.called || result.OutputPath != filepath.Join(workingDir, "daily-2026-05-29.md") || result.ValidationStatus != promptexec.ValidationPassed || result.ProfileID == "" || result.BackendID == "" || result.ModelName == "" {
|
|
t.Fatalf("result = %#v", result)
|
|
}
|
|
if result.LLMDebugPath != "" {
|
|
t.Fatalf("unexpected debug output = %q", result.LLMDebugPath)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(workingDir, "workspace")); !os.IsNotExist(err) {
|
|
t.Fatalf("unexpected default state directory: %v", err)
|
|
}
|
|
data, err := os.ReadFile(result.OutputPath)
|
|
if err != nil || len(data) == 0 {
|
|
t.Fatalf("output = %q, error = %v", data, err)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedReturnsResolvedResultWhenCollectionFails(t *testing.T) {
|
|
cfg := config.Defaults()
|
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
|
collectionErr := errors.New("weather source unavailable")
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{
|
|
Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
|
WorkingDir: t.TempDir(), Collector: &generationCollector{err: collectionErr}, Executor: &generationExecutor{},
|
|
})
|
|
if !errors.Is(err, collectionErr) {
|
|
t.Fatalf("GenerateDetailed() error = %v, want %v", err, collectionErr)
|
|
}
|
|
if result == nil || result.ReportID != report.Daily || result.RunID == "" || result.ProfileID != "fixture" || result.BackendID != "fixture" || result.ModelName != "fixture-model" || result.OutputPath != "" {
|
|
t.Fatalf("result = %#v", result)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedInspectsPromptBeforeCollectingWeather(t *testing.T) {
|
|
cfg := generationConfig()
|
|
inspectionErr := errors.New("profile is invalid")
|
|
collector := &generationCollector{bundle: generationBundlePointer(t)}
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), Collector: collector, Executor: &generationExecutor{inspectErr: inspectionErr}})
|
|
if !errors.Is(err, inspectionErr) || collector.called || result == nil {
|
|
t.Fatalf("GenerateDetailed() result/error/collector-called = %#v/%v/%t", result, err, collector.called)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedPreservesDestinationBeforePublish(t *testing.T) {
|
|
for _, scenario := range []struct {
|
|
name string
|
|
executor generationExecutor
|
|
cancel bool
|
|
}{
|
|
{name: "generation", executor: generationExecutor{executeErr: errors.New("provider unavailable")}},
|
|
{name: "render", executor: generationExecutor{rawOutput: []byte(`{"summary":""}`)}},
|
|
{name: "cancellation", executor: generationExecutor{respectCancellation: true}, cancel: true},
|
|
} {
|
|
t.Run(scenario.name, func(t *testing.T) {
|
|
outputPath := filepath.Join(t.TempDir(), "daily.md")
|
|
if err := os.WriteFile(outputPath, []byte("previous report"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
if scenario.cancel {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithCancel(ctx)
|
|
cancel()
|
|
}
|
|
bundle := generationBundle(t)
|
|
result, err := GenerateDetailed(ctx, GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &scenario.executor})
|
|
data, readErr := os.ReadFile(outputPath)
|
|
if err == nil || result == nil || readErr != nil || string(data) != "previous report" {
|
|
t.Fatalf("GenerateDetailed() result/error/output = %#v/%v/%q (%v)", result, err, data, readErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedRetainsPublishedOutputWhenNotificationFails(t *testing.T) {
|
|
cfg := generationConfig()
|
|
cfg.Notify.Distributor.Enabled = true
|
|
cfg.Notify.Distributor.PipelineIDTemplate = "weather"
|
|
bundle := generationBundle(t)
|
|
outputPath := filepath.Join(t.TempDir(), "daily.md")
|
|
notifier := &generationNotifier{err: errors.New("distributor unavailable")}
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: cfg, Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, Notifier: notifier})
|
|
if err == nil || result == nil || result.OutputPath != outputPath || notifier.request.ReportPath != outputPath || len(notifier.request.BundlePaths) == 0 {
|
|
t.Fatalf("GenerateDetailed() result/error/request = %#v/%v/%#v", result, err, notifier.request)
|
|
}
|
|
if data, readErr := os.ReadFile(outputPath); readErr != nil || len(data) == 0 {
|
|
t.Fatalf("published output = %q, error = %v", data, readErr)
|
|
}
|
|
}
|
|
|
|
func TestGenerateDetailedDoesNotReplaceDirectoryOutput(t *testing.T) {
|
|
bundle := generationBundle(t)
|
|
outputPath := filepath.Join(t.TempDir(), "daily.md")
|
|
if err := os.Mkdir(outputPath, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := GenerateDetailed(context.Background(), GenerateRequest{Config: generationConfig(), Report: ReportDaily, Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"), WorkingDir: t.TempDir(), OutputPath: outputPath, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}})
|
|
info, statErr := os.Stat(outputPath)
|
|
if err == nil || result == nil || statErr != nil || !info.IsDir() {
|
|
t.Fatalf("GenerateDetailed() result/error/output-info = %#v/%v/%#v (%v)", result, err, info, statErr)
|
|
}
|
|
}
|
|
|
|
func generationConfig() config.Config {
|
|
cfg := config.Defaults()
|
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
|
return cfg
|
|
}
|
|
|
|
func generationBundlePointer(t *testing.T) *weatherdata.Bundle {
|
|
bundle := generationBundle(t)
|
|
return &bundle
|
|
}
|
|
|
|
type generationNotifier struct {
|
|
err error
|
|
request NotificationRequest
|
|
batchRequest batchNotificationRequest
|
|
batchCalls int
|
|
}
|
|
|
|
func (n *generationNotifier) Notify(_ context.Context, request NotificationRequest) (*NotificationResult, error) {
|
|
n.request = request
|
|
return nil, n.err
|
|
}
|
|
|
|
func (n *generationNotifier) NotifyBatch(_ context.Context, request batchNotificationRequest) (*NotificationResult, error) {
|
|
n.batchCalls++
|
|
n.batchRequest = request
|
|
for _, file := range request.Files {
|
|
if _, err := os.Stat(file.SourcePath); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return &NotificationResult{Status: "succeeded", PipelineID: request.PipelineID, BundleID: request.BundleID}, nil
|
|
}
|
|
|
|
func generationBundle(t *testing.T) weatherdata.Bundle {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join("..", "forecast", "testdata", "daily_bundle.json"))
|
|
if err != nil {
|
|
t.Fatalf("read bundle fixture: %v", err)
|
|
}
|
|
var bundle weatherdata.Bundle
|
|
if err := json.Unmarshal(data, &bundle); err != nil {
|
|
t.Fatalf("decode bundle fixture: %v", err)
|
|
}
|
|
return bundle
|
|
}
|
|
func generationTime(value string) time.Time {
|
|
parsed, _ := time.Parse(time.RFC3339, value)
|
|
return parsed
|
|
}
|
|
|
|
var _ promptexec.Executor = (*generationExecutor)(nil)
|
|
var _ Collector = (*generationCollector)(nil)
|
|
var _ Notifier = (*generationNotifier)(nil)
|
|
var _ = report.Daily
|