400 lines
18 KiB
Go
400 lines
18 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) {
|
|
cfg := comparisonConfig()
|
|
bundle := generationBundle(t)
|
|
workingDir := t.TempDir()
|
|
executor := &generationExecutor{}
|
|
inspectedBeforeCollection := false
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: &generationCollector{bundle: &bundle, beforeRun: func() {
|
|
inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2
|
|
}}, Executor: executor,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("CompareDetailed() error = %v", err)
|
|
}
|
|
if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" {
|
|
t.Fatalf("result/executor = %#v/%#v", result, executor)
|
|
}
|
|
if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
|
t.Fatalf("published paths = %#v", result)
|
|
}
|
|
for index, profile := range result.Results {
|
|
if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil {
|
|
t.Fatalf("profile result = %#v", profile)
|
|
}
|
|
}
|
|
data, readErr := os.ReadFile(result.ManifestPath)
|
|
if readErr != nil {
|
|
t.Fatal(readErr)
|
|
}
|
|
var manifest comparison.Manifest
|
|
if err := json.Unmarshal(data, &manifest); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 {
|
|
t.Fatalf("manifest = %#v", manifest)
|
|
}
|
|
if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) {
|
|
t.Fatalf("manifest report paths = %#v", manifest.Results)
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) {
|
|
cfg := comparisonConfig()
|
|
bundle := generationBundle(t)
|
|
executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"},
|
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
|
})
|
|
if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
|
}
|
|
failure := result.Results[1]
|
|
if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") {
|
|
t.Fatalf("failure = %#v", failure)
|
|
}
|
|
if _, statErr := os.Stat(result.ManifestPath); statErr != nil {
|
|
t.Fatalf("partial manifest: %v", statErr)
|
|
}
|
|
if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil {
|
|
t.Fatalf("successful partial report: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedRetainsCommittedPathsWhenBackupCleanupFails(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
state comparison.BackupRecoveryState
|
|
path bool
|
|
}{
|
|
{name: "complete recovery bundle", state: comparison.BackupRecoveryComplete, path: true},
|
|
{name: "partial remnants", state: comparison.BackupRecoveryPartial, path: true},
|
|
{name: "absent backup", state: comparison.BackupRecoveryAbsent},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
bundle := generationBundle(t)
|
|
recoveryPath := ""
|
|
if test.path {
|
|
recoveryPath = filepath.Join(t.TempDir(), ".comparison-daily.backup-recovery")
|
|
}
|
|
cleanupCause := errors.New("backup cleanup failed")
|
|
publish := func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error) {
|
|
return comparison.PublicationResult{Committed: true, RecoveryState: test.state, RecoveryPath: recoveryPath}, &comparison.PublicationCleanupError{RecoveryState: test.state, RecoveryPath: recoveryPath, Err: cleanupCause}
|
|
}
|
|
|
|
result, err := compareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
|
}, publish)
|
|
var cleanupErr *comparison.PublicationCleanupError
|
|
if result == nil || !errors.As(err, &cleanupErr) || !errors.Is(err, cleanupCause) || cleanupErr.RecoveryState != test.state || cleanupErr.RecoveryPath != recoveryPath || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
|
}
|
|
for _, profile := range result.Results {
|
|
if profile.Status == comparison.StatusSucceeded && !filepath.IsAbs(profile.ReportPath) {
|
|
t.Fatalf("published profile result = %#v", profile)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) {
|
|
invalidDestination := filepath.Join(t.TempDir(), "not-a-directory")
|
|
if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bundle := generationBundle(t)
|
|
collector := &generationCollector{bundle: &bundle}
|
|
executor := &generationExecutor{}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor,
|
|
})
|
|
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" {
|
|
t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor)
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedFinalizesUnpublishedFailures(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
prepare func(t *testing.T, outputDirectory string)
|
|
debugDir string
|
|
executor *generationExecutor
|
|
collector *generationCollector
|
|
wantPrompt bool
|
|
wantCollection bool
|
|
}{
|
|
{
|
|
name: "destination preflight",
|
|
prepare: func(t *testing.T, outputDirectory string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(outputDirectory, []byte("not a directory"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
},
|
|
executor: &generationExecutor{},
|
|
},
|
|
{
|
|
name: "debug initialization",
|
|
debugDir: "relative-debug-directory",
|
|
executor: &generationExecutor{},
|
|
collector: &generationCollector{},
|
|
},
|
|
{
|
|
name: "prompt preflight",
|
|
executor: &generationExecutor{inspectErr: promptexec.NewError(promptexec.PromptLoad, "unsafe prompt detail", errors.New("unsafe cause"))},
|
|
collector: &generationCollector{},
|
|
wantPrompt: false,
|
|
},
|
|
{
|
|
name: "profile preflight",
|
|
executor: &generationExecutor{profileInspectErrors: map[string]error{
|
|
"weather-deep": promptexec.NewError(promptexec.MissingCredential, "profile credential is unavailable", errors.New("unsafe cause")),
|
|
}},
|
|
collector: &generationCollector{},
|
|
wantPrompt: true,
|
|
},
|
|
{
|
|
name: "collection",
|
|
executor: &generationExecutor{},
|
|
collector: &generationCollector{err: errors.New("collection failed")},
|
|
wantPrompt: true,
|
|
wantCollection: true,
|
|
},
|
|
{
|
|
name: "preparation",
|
|
executor: &generationExecutor{},
|
|
collector: &generationCollector{bundle: &weatherdata.Bundle{}},
|
|
wantPrompt: true,
|
|
wantCollection: true,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
workingDirectory := t.TempDir()
|
|
outputDirectory := filepath.Join(workingDirectory, "comparison-output")
|
|
if test.prepare != nil {
|
|
test.prepare(t, outputDirectory)
|
|
}
|
|
collector := test.collector
|
|
if collector == nil {
|
|
bundle := generationBundle(t)
|
|
collector = &generationCollector{bundle: &bundle}
|
|
}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: workingDirectory, OutputDir: outputDirectory, LLMDebugDir: test.debugDir,
|
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: collector, Executor: test.executor,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("CompareDetailed() error = nil")
|
|
}
|
|
assertUnpublishedComparisonResult(t, result, outputDirectory)
|
|
if (result.PromptID != "") != test.wantPrompt || (result.PromptHash != "") != test.wantPrompt {
|
|
t.Fatalf("prompt identity = %q/%q, want resolved=%t", result.PromptID, result.PromptHash, test.wantPrompt)
|
|
}
|
|
if collector.called != test.wantCollection || test.executor.executeCalls != 0 {
|
|
t.Fatalf("collection/execution = %t/%d, want collection=%t and no execution", collector.called, test.executor.executeCalls, test.wantCollection)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedLeavesDestinationWhenCollectionOrPreparationFails(t *testing.T) {
|
|
collectionErr := errors.New("weather collection failed")
|
|
for _, test := range []struct {
|
|
name string
|
|
collector *generationCollector
|
|
}{
|
|
{name: "collection", collector: &generationCollector{err: collectionErr}},
|
|
{name: "preparation", collector: &generationCollector{bundle: &weatherdata.Bundle{}}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
executor := &generationExecutor{}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: test.collector, Executor: executor,
|
|
})
|
|
if err == nil || result == nil || executor.executeCalls != 0 || result.ManifestPath != "" || result.DataPackagePath != "" {
|
|
t.Fatalf("CompareDetailed() result/error/executor = %#v/%v/%#v", result, err, executor)
|
|
}
|
|
if _, statErr := os.Stat(filepath.Join(workingDir, "comparison-daily-2026-05-29")); !os.IsNotExist(statErr) {
|
|
t.Fatalf("comparison destination stat error = %v", statErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) {
|
|
bundle := generationBundle(t)
|
|
executor := &generationExecutor{executeErrors: map[string]error{
|
|
"weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"),
|
|
}}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
|
})
|
|
if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" {
|
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
|
}
|
|
for _, profile := range result.Results {
|
|
if profile.ReportPath != "" || profile.Error == nil {
|
|
t.Fatalf("failed profile = %#v", profile)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
bundle := generationBundle(t)
|
|
request := ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
|
}
|
|
previous, err := CompareDetailed(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("initial CompareDetailed() error = %v", err)
|
|
}
|
|
before, err := os.ReadFile(previous.ManifestPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
request.Replace = true
|
|
request.Executor = &generationExecutor{cancelBeforeReturn: cancel}
|
|
result, err := CompareDetailed(ctx, request)
|
|
if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" {
|
|
t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err)
|
|
}
|
|
after, readErr := os.ReadFile(previous.ManifestPath)
|
|
if readErr != nil || string(after) != string(before) {
|
|
t.Fatalf("published manifest changed = %q, error = %v", after, readErr)
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedPreservesCompletedProfileFailureWhenCanceled(t *testing.T) {
|
|
bundle := generationBundle(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
failureStarted := make(chan struct{})
|
|
var signalFailure sync.Once
|
|
executor := &generationExecutor{
|
|
validations: map[string]promptexec.ValidationStatus{"weather-light": promptexec.ValidationFailed},
|
|
waitForCancellation: map[string]bool{"weather-deep": true},
|
|
beforeExecute: func(request promptexec.ExecuteRequest) {
|
|
if request.ProfileID == "weather-light" {
|
|
signalFailure.Do(func() { close(failureStarted) })
|
|
}
|
|
},
|
|
}
|
|
results := make(chan struct {
|
|
result *ComparisonResult
|
|
err error
|
|
}, 1)
|
|
go func() {
|
|
result, err := CompareDetailed(ctx, ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
|
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
|
})
|
|
results <- struct {
|
|
result *ComparisonResult
|
|
err error
|
|
}{result: result, err: err}
|
|
}()
|
|
|
|
select {
|
|
case <-failureStarted:
|
|
cancel()
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("timed out waiting for the completed profile failure")
|
|
}
|
|
completed := <-results
|
|
if !errors.Is(completed.err, context.Canceled) || completed.result == nil || completed.result.ManifestPath != "" || completed.result.DataPackagePath != "" || completed.result.Succeeded != 0 || completed.result.Failed != 2 {
|
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", completed.result, completed.err)
|
|
}
|
|
failed, canceled := completed.result.Results[0], completed.result.Results[1]
|
|
if failed.Error == nil || failed.Error.Category != string(promptexec.ValidationRejected) || failed.ValidationStatus != promptexec.ValidationFailed || failed.ReportPath != "" {
|
|
t.Fatalf("completed failure = %#v", failed)
|
|
}
|
|
if canceled.Error == nil || canceled.Error.Category != string(promptexec.Canceled) || canceled.ValidationStatus != promptexec.ValidationSkipped || canceled.ReportPath != "" {
|
|
t.Fatalf("canceled profile = %#v", canceled)
|
|
}
|
|
}
|
|
|
|
func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) {
|
|
workingDir := t.TempDir()
|
|
target := filepath.Join(workingDir, "comparison-output")
|
|
bundle := generationBundle(t)
|
|
executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) {
|
|
_ = os.WriteFile(target, []byte("changed"), 0o600)
|
|
}}
|
|
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
|
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
|
WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
|
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
|
})
|
|
if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" {
|
|
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
|
}
|
|
data, readErr := os.ReadFile(target)
|
|
if readErr != nil || string(data) != "changed" {
|
|
t.Fatalf("destination = %q, error = %v", data, readErr)
|
|
}
|
|
}
|
|
|
|
func comparisonConfig() config.Config {
|
|
cfg := config.Defaults()
|
|
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
|
return cfg
|
|
}
|
|
|
|
func assertUnpublishedComparisonResult(t *testing.T, result *ComparisonResult, outputDirectory string) {
|
|
t.Helper()
|
|
if result == nil || result.OutputDirectory != outputDirectory || !filepath.IsAbs(result.OutputDirectory) || result.FinishedAt.IsZero() || result.FinishedAt.Location() != time.UTC || result.FinishedAt.Before(result.StartedAt) || result.ManifestPath != "" || result.DataPackagePath != "" {
|
|
t.Fatalf("unpublished comparison result = %#v", result)
|
|
}
|
|
for _, profile := range result.Results {
|
|
if profile.ReportPath != "" {
|
|
t.Fatalf("unpublished profile result = %#v", profile)
|
|
}
|
|
}
|
|
}
|