Assemble comparison application workflow
This commit is contained in:
214
internal/app/comparison.go
Normal file
214
internal/app/comparison.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
// ComparisonRequest describes one explicit, multi-profile report comparison.
|
||||
// It deliberately does not accept a notifier: comparison publication is local.
|
||||
type ComparisonRequest struct {
|
||||
Config config.Config
|
||||
Report ReportKind
|
||||
ProfileIDs []string
|
||||
WorkingDir string
|
||||
OutputDir string
|
||||
Replace bool
|
||||
LLMDebugDir string
|
||||
Date time.Time
|
||||
Clock timeutil.Clock
|
||||
Collector Collector
|
||||
Executor promptexec.Executor
|
||||
}
|
||||
|
||||
// ComparisonResult records the resolved comparison and profile outcomes.
|
||||
type ComparisonResult struct {
|
||||
ComparisonID string
|
||||
ReportID report.ID
|
||||
ReportName string
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Timezone string
|
||||
ValidPeriod timeutil.Period
|
||||
OutputDirectory string
|
||||
ManifestPath string
|
||||
DataPackagePath string
|
||||
Total int
|
||||
Succeeded int
|
||||
Failed int
|
||||
Results []ComparisonProfileResult
|
||||
}
|
||||
|
||||
// ComparisonProfileResult records one explicitly selected profile.
|
||||
type ComparisonProfileResult struct {
|
||||
Position int
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
Status string
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
ReportPath string
|
||||
LLMDebugPath string
|
||||
Error *comparison.SafeError
|
||||
}
|
||||
|
||||
// CompareDetailed assembles, executes, and atomically publishes a comparison
|
||||
// bundle. Profile failures publish a complete partial bundle; all other
|
||||
// failures leave the destination untouched.
|
||||
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
|
||||
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clock := req.Clock
|
||||
if clock == nil {
|
||||
clock = timeutil.SystemClock{}
|
||||
}
|
||||
now := clock.Now()
|
||||
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata := resolved.Metadata()
|
||||
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build comparison identity: %w", err)
|
||||
}
|
||||
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
|
||||
|
||||
outputName, err := resolved.OutputName()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("resolve comparison output name: %w", err)
|
||||
}
|
||||
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
initialPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("preflight comparison destination: %w", err)
|
||||
}
|
||||
result.OutputDirectory = initialPlan.Target
|
||||
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||
if err != nil {
|
||||
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
}
|
||||
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
|
||||
})
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
|
||||
|
||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("prepare comparison report: %w", err)
|
||||
}
|
||||
|
||||
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
|
||||
})
|
||||
result.FinishedAt = clock.Now().UTC()
|
||||
copyComparisonOutcomes(result, executed.Outcomes, false)
|
||||
if executed.Canceled {
|
||||
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
|
||||
}
|
||||
|
||||
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
|
||||
if err := bundle.Validate(); err != nil {
|
||||
return result, fmt.Errorf("build comparison bundle: %w", err)
|
||||
}
|
||||
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("re-preflight comparison destination: %w", err)
|
||||
}
|
||||
if err := comparison.Publish(ctx, publicationPlan, bundle); err != nil {
|
||||
return result, fmt.Errorf("publish comparison bundle: %w", err)
|
||||
}
|
||||
result.OutputDirectory = publicationPlan.Target
|
||||
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
|
||||
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
|
||||
copyComparisonOutcomes(result, executed.Outcomes, true)
|
||||
|
||||
if result.Failed > 0 {
|
||||
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
|
||||
metadata := resolved.Metadata()
|
||||
return &ComparisonResult{
|
||||
ComparisonID: comparisonID,
|
||||
ReportID: resolved.Definition.ID,
|
||||
ReportName: resolved.Definition.Name,
|
||||
StartedAt: startedAt,
|
||||
Timezone: req.Config.WeatherAPI.Timezone,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
}
|
||||
}
|
||||
|
||||
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
|
||||
result.Results = make([]ComparisonProfileResult, len(outcomes))
|
||||
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
|
||||
for index, outcome := range outcomes {
|
||||
profile := ComparisonProfileResult{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
|
||||
}
|
||||
if published && outcome.Status == comparison.StatusSucceeded {
|
||||
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
|
||||
}
|
||||
result.Results[index] = profile
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
result.Succeeded++
|
||||
} else {
|
||||
result.Failed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
|
||||
manifest := comparison.Manifest{
|
||||
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
|
||||
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
|
||||
ReportID: string(result.ReportID), Timezone: result.Timezone,
|
||||
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
|
||||
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
|
||||
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
|
||||
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
|
||||
Results: make([]comparison.Result, len(outcomes)),
|
||||
}
|
||||
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
|
||||
for index, outcome := range outcomes {
|
||||
manifestResult := comparison.Result{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
|
||||
}
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
manifestResult.ReportPath = outcome.ReportPath
|
||||
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
|
||||
}
|
||||
bundle.Manifest.Results[index] = manifestResult
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
Reference in New Issue
Block a user