diff --git a/internal/app/comparison.go b/internal/app/comparison.go new file mode 100644 index 0000000..a81e328 --- /dev/null +++ b/internal/app/comparison.go @@ -0,0 +1,214 @@ +package app + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" + "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" +) + +// ComparisonRequest describes one explicit, multi-profile report comparison. +// It deliberately does not accept a notifier: comparison publication is local. +type ComparisonRequest struct { + Config config.Config + Report ReportKind + ProfileIDs []string + WorkingDir string + OutputDir string + Replace bool + LLMDebugDir string + Date time.Time + Clock timeutil.Clock + Collector Collector + Executor promptexec.Executor +} + +// ComparisonResult records the resolved comparison and profile outcomes. +type ComparisonResult struct { + ComparisonID string + ReportID report.ID + ReportName string + PromptID string + PromptVersion string + PromptHash string + StartedAt time.Time + FinishedAt time.Time + Timezone string + ValidPeriod timeutil.Period + OutputDirectory string + ManifestPath string + DataPackagePath string + Total int + Succeeded int + Failed int + Results []ComparisonProfileResult +} + +// ComparisonProfileResult records one explicitly selected profile. +type ComparisonProfileResult struct { + Position int + ProfileID string + BackendID string + ModelName string + Status string + ValidationStatus promptexec.ValidationStatus + ReportPath string + LLMDebugPath string + Error *comparison.SafeError +} + +// CompareDetailed assembles, executes, and atomically publishes a comparison +// bundle. Profile failures publish a complete partial bundle; all other +// failures leave the destination untouched. +func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) { + if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil { + return nil, err + } + + clock := req.Clock + if clock == nil { + clock = timeutil.SystemClock{} + } + now := clock.Now() + resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now) + if err != nil { + return nil, err + } + metadata := resolved.Metadata() + comparisonID, err := comparison.BuildComparisonID(metadata.RunID) + if err != nil { + return nil, fmt.Errorf("build comparison identity: %w", err) + } + result := initialComparisonResult(req, resolved, comparisonID, now.UTC()) + + outputName, err := resolved.OutputName() + if err != nil { + return result, fmt.Errorf("resolve comparison output name: %w", err) + } + outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName) + if err != nil { + return result, err + } + initialPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace) + if err != nil { + return result, fmt.Errorf("preflight comparison destination: %w", err) + } + result.OutputDirectory = initialPlan.Target + + debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir) + if err != nil { + return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err) + } + inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{ + Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv, + }) + if err != nil { + return result, err + } + result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash + + collection, err := collectWeather(ctx, req.Config, req.Collector) + if err != nil { + return result, err + } + prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection}) + if err != nil { + return result, fmt.Errorf("prepare comparison report: %w", err) + } + + executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{ + Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor, + }) + result.FinishedAt = clock.Now().UTC() + copyComparisonOutcomes(result, executed.Outcomes, false) + if executed.Canceled { + return result, fmt.Errorf("comparison execution: %w", ctx.Err()) + } + + bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes) + if err := bundle.Validate(); err != nil { + return result, fmt.Errorf("build comparison bundle: %w", err) + } + publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace) + if err != nil { + return result, fmt.Errorf("re-preflight comparison destination: %w", err) + } + if err := comparison.Publish(ctx, publicationPlan, bundle); err != nil { + return result, fmt.Errorf("publish comparison bundle: %w", err) + } + result.OutputDirectory = publicationPlan.Target + result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename) + result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename) + copyComparisonOutcomes(result, executed.Outcomes, true) + + if result.Failed > 0 { + return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed) + } + return result, nil +} + +func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult { + metadata := resolved.Metadata() + return &ComparisonResult{ + ComparisonID: comparisonID, + ReportID: resolved.Definition.ID, + ReportName: resolved.Definition.Name, + StartedAt: startedAt, + Timezone: req.Config.WeatherAPI.Timezone, + ValidPeriod: metadata.ValidPeriod, + } +} + +func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) { + result.Results = make([]ComparisonProfileResult, len(outcomes)) + result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0 + for index, outcome := range outcomes { + profile := ComparisonProfileResult{ + Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName, + Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error, + } + if published && outcome.Status == comparison.StatusSucceeded { + profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath) + } + result.Results[index] = profile + if outcome.Status == comparison.StatusSucceeded { + result.Succeeded++ + } else { + result.Failed++ + } + } +} + +func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle { + manifest := comparison.Manifest{ + SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID, + StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(), + ReportID: string(result.ReportID), Timezone: result.Timezone, + ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End}, + PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash, + DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)}, + Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed, + Results: make([]comparison.Result, len(outcomes)), + } + bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage} + for index, outcome := range outcomes { + manifestResult := comparison.Result{ + Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName, + Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error, + } + if outcome.Status == comparison.StatusSucceeded { + manifestResult.ReportPath = outcome.ReportPath + bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown}) + } + bundle.Manifest.Results[index] = manifestResult + } + return bundle +} diff --git a/internal/app/comparison_test.go b/internal/app/comparison_test.go new file mode 100644 index 0000000..e1784a0 --- /dev/null +++ b/internal/app/comparison_test.go @@ -0,0 +1,180 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" +) + +func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) { + cfg := comparisonConfig() + bundle := generationBundle(t) + workingDir := t.TempDir() + executor := &generationExecutor{} + inspectedBeforeCollection := false + result, err := CompareDetailed(context.Background(), ComparisonRequest{ + Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"}, + WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, + Collector: &generationCollector{bundle: &bundle, beforeRun: func() { + inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2 + }}, Executor: executor, + }) + if err != nil { + t.Fatalf("CompareDetailed() error = %v", err) + } + if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" { + t.Fatalf("result/executor = %#v/%#v", result, executor) + } + if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) { + t.Fatalf("published paths = %#v", result) + } + for index, profile := range result.Results { + if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil { + t.Fatalf("profile result = %#v", profile) + } + } + data, readErr := os.ReadFile(result.ManifestPath) + if readErr != nil { + t.Fatal(readErr) + } + var manifest comparison.Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 { + t.Fatalf("manifest = %#v", manifest) + } + if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) { + t.Fatalf("manifest report paths = %#v", manifest.Results) + } +} + +func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) { + cfg := comparisonConfig() + bundle := generationBundle(t) + executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}} + result, err := CompareDetailed(context.Background(), ComparisonRequest{ + Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"}, + WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, + Collector: &generationCollector{bundle: &bundle}, Executor: executor, + }) + if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 { + t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err) + } + failure := result.Results[1] + if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") { + t.Fatalf("failure = %#v", failure) + } + if _, statErr := os.Stat(result.ManifestPath); statErr != nil { + t.Fatalf("partial manifest: %v", statErr) + } + if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil { + t.Fatalf("successful partial report: %v", statErr) + } +} + +func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) { + invalidDestination := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + bundle := generationBundle(t) + collector := &generationCollector{bundle: &bundle} + executor := &generationExecutor{} + result, err := CompareDetailed(context.Background(), ComparisonRequest{ + Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"}, + WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor, + }) + if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" { + t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor) + } +} + +func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) { + bundle := generationBundle(t) + executor := &generationExecutor{executeErrors: map[string]error{ + "weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"), + }} + result, err := CompareDetailed(context.Background(), ComparisonRequest{ + Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"}, + WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, + Collector: &generationCollector{bundle: &bundle}, Executor: executor, + }) + if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" { + t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err) + } + for _, profile := range result.Results { + if profile.ReportPath != "" || profile.Error == nil { + t.Fatalf("failed profile = %#v", profile) + } + } +} + +func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) { + workingDir := t.TempDir() + bundle := generationBundle(t) + request := ComparisonRequest{ + Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"}, + WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{}, + } + previous, err := CompareDetailed(context.Background(), request) + if err != nil { + t.Fatalf("initial CompareDetailed() error = %v", err) + } + before, err := os.ReadFile(previous.ManifestPath) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + request.Replace = true + request.Executor = &generationExecutor{cancelBeforeReturn: cancel} + result, err := CompareDetailed(ctx, request) + if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" { + t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err) + } + after, readErr := os.ReadFile(previous.ManifestPath) + if readErr != nil || string(after) != string(before) { + t.Fatalf("published manifest changed = %q, error = %v", after, readErr) + } +} + +func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) { + workingDir := t.TempDir() + target := filepath.Join(workingDir, "comparison-output") + bundle := generationBundle(t) + executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) { + _ = os.WriteFile(target, []byte("changed"), 0o600) + }} + result, err := CompareDetailed(context.Background(), ComparisonRequest{ + Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"}, + WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"), + Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor, + }) + if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" { + t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err) + } + data, readErr := os.ReadFile(target) + if readErr != nil || string(data) != "changed" { + t.Fatalf("destination = %q, error = %v", data, readErr) + } +} + +func comparisonConfig() config.Config { + cfg := config.Defaults() + cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home" + return cfg +} diff --git a/internal/app/generation_test.go b/internal/app/generation_test.go index db19d8e..63580c4 100644 --- a/internal/app/generation_test.go +++ b/internal/app/generation_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "sync" "testing" "time" @@ -37,52 +38,77 @@ type generationExecutor struct { called bool executeCalls int promptInspections int + profileInspections int inspectErr error executeErr error + executeErrors map[string]error + beforeExecute func(promptexec.ExecuteRequest) cancelBeforeReturn context.CancelFunc validation promptexec.ValidationStatus rawOutput []byte failedPrompt string } +var generationExecutorMu sync.Mutex + func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) { + generationExecutorMu.Lock() + defer generationExecutorMu.Unlock() e.promptInspections++ if e.inspectErr != nil { return promptexec.PromptInspection{}, e.inspectErr } definition := generationDefinitionForPrompt(id) - return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil + return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil } -func (*generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { +func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { + generationExecutorMu.Lock() + defer generationExecutorMu.Unlock() + e.profileInspections++ return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil } func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) { stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) - if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil { + if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil { return nil, err } + generationExecutorMu.Lock() e.called = true e.executeCalls++ - if e.executeErr != nil { - return nil, e.executeErr - } + beforeExecute := e.beforeExecute + profileErr := e.executeErrors[req.ProfileID] + executeErr := e.executeErr status := e.validation + rawOutput := append([]byte(nil), e.rawOutput...) + failedPrompt := e.failedPrompt + cancelBeforeReturn := e.cancelBeforeReturn + generationExecutorMu.Unlock() + if beforeExecute != nil { + beforeExecute(req) + } + if profileErr != nil { + return nil, profileErr + } + if executeErr != nil { + return nil, executeErr + } if status == "" { status = promptexec.ValidationPassed } - if e.failedPrompt == req.PromptID { + if failedPrompt == req.PromptID { status = promptexec.ValidationFailed } - rawOutput := e.rawOutput if rawOutput == nil { rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`) } - if e.cancelBeforeReturn != nil { - e.cancelBeforeReturn() + if cancelBeforeReturn != nil { + cancelBeforeReturn() } - return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil + return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil } +const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + func generationDefinitionForPrompt(promptID string) report.Definition { for _, definition := range report.DefaultRegistry().All() { if definition.PromptID == promptID {