238 lines
8.9 KiB
Go
238 lines
8.9 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
)
|
|
|
|
// ComparisonRequest describes one explicit, multi-profile report comparison.
|
|
// It deliberately does not accept a notifier: comparison publication is local.
|
|
type ComparisonRequest struct {
|
|
Config config.Config
|
|
Report ReportKind
|
|
ProfileIDs []string
|
|
WorkingDir string
|
|
OutputDir string
|
|
Replace bool
|
|
LLMDebugDir string
|
|
Date time.Time
|
|
Clock timeutil.Clock
|
|
Collector Collector
|
|
Executor promptexec.Executor
|
|
}
|
|
|
|
// ComparisonResult records the resolved comparison and profile outcomes.
|
|
type ComparisonResult struct {
|
|
ComparisonID string
|
|
ReportID report.ID
|
|
ReportName string
|
|
PromptID string
|
|
PromptVersion string
|
|
PromptHash string
|
|
StartedAt time.Time
|
|
FinishedAt time.Time
|
|
Timezone string
|
|
ValidPeriod timeutil.Period
|
|
OutputDirectory string
|
|
ManifestPath string
|
|
DataPackagePath string
|
|
Total int
|
|
Succeeded int
|
|
Failed int
|
|
Results []ComparisonProfileResult
|
|
}
|
|
|
|
// ComparisonProfileResult records one explicitly selected profile.
|
|
type ComparisonProfileResult struct {
|
|
Position int
|
|
ProfileID string
|
|
BackendID string
|
|
ModelName string
|
|
Status string
|
|
ValidationStatus promptexec.ValidationStatus
|
|
ReportPath string
|
|
LLMDebugPath string
|
|
Error *comparison.SafeError
|
|
}
|
|
|
|
type comparisonPublisher func(context.Context, comparison.DestinationPlan, comparison.LogicalBundle) (comparison.PublicationResult, error)
|
|
|
|
// CompareDetailed assembles, executes, and atomically publishes a comparison
|
|
// bundle. Profile failures publish a complete partial bundle. Failures before
|
|
// commit leave the destination untouched; a post-commit cleanup failure leaves
|
|
// the new bundle installed and returns its artifact paths with an error.
|
|
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
|
|
return compareDetailed(ctx, req, comparison.Publish)
|
|
}
|
|
|
|
func compareDetailed(ctx context.Context, req ComparisonRequest, publish comparisonPublisher) (*ComparisonResult, error) {
|
|
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
clock := req.Clock
|
|
if clock == nil {
|
|
clock = timeutil.SystemClock{}
|
|
}
|
|
now := clock.Now()
|
|
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
metadata := resolved.Metadata()
|
|
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build comparison identity: %w", err)
|
|
}
|
|
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
|
|
defer func() {
|
|
if result.FinishedAt.IsZero() {
|
|
finalizeComparisonResult(result, clock)
|
|
}
|
|
}()
|
|
|
|
outputName, err := resolved.OutputName()
|
|
if err != nil {
|
|
return result, fmt.Errorf("resolve comparison output name: %w", err)
|
|
}
|
|
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
result.OutputDirectory = outputDirectory
|
|
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
|
if err != nil {
|
|
return result, fmt.Errorf("preflight comparison destination: %w", err)
|
|
}
|
|
|
|
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
|
if err != nil {
|
|
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
|
}
|
|
defer func() { _ = debugWriter.Close() }()
|
|
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
|
|
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
|
|
})
|
|
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
|
|
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection, handler: inspection.handler})
|
|
if err != nil {
|
|
return result, fmt.Errorf("prepare comparison report: %w", err)
|
|
}
|
|
|
|
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
|
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
|
|
})
|
|
finalizeComparisonResult(result, clock)
|
|
copyComparisonOutcomes(result, executed.Outcomes, false)
|
|
if executed.Canceled {
|
|
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
|
|
}
|
|
|
|
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
|
|
if err := bundle.Validate(); err != nil {
|
|
return result, fmt.Errorf("build comparison bundle: %w", err)
|
|
}
|
|
publication, err := publish(ctx, publicationPlan, bundle)
|
|
if publication.Committed {
|
|
result.OutputDirectory = publicationPlan.Target
|
|
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
|
|
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
|
|
copyComparisonOutcomes(result, executed.Outcomes, true)
|
|
}
|
|
if err != nil {
|
|
return result, fmt.Errorf("publish comparison bundle: %w", err)
|
|
}
|
|
|
|
if result.Failed > 0 {
|
|
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func finalizeComparisonResult(result *ComparisonResult, clock timeutil.Clock) {
|
|
finishedAt := clock.Now().UTC()
|
|
if finishedAt.IsZero() {
|
|
finishedAt = time.Unix(0, 1).UTC()
|
|
}
|
|
if finishedAt.Before(result.StartedAt) {
|
|
finishedAt = result.StartedAt
|
|
}
|
|
result.FinishedAt = finishedAt
|
|
}
|
|
|
|
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
|
|
metadata := resolved.Metadata()
|
|
return &ComparisonResult{
|
|
ComparisonID: comparisonID,
|
|
ReportID: resolved.Definition.ID,
|
|
ReportName: resolved.Definition.Name,
|
|
StartedAt: startedAt,
|
|
Timezone: req.Config.WeatherAPI.Timezone,
|
|
ValidPeriod: metadata.ValidPeriod,
|
|
}
|
|
}
|
|
|
|
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
|
|
result.Results = make([]ComparisonProfileResult, len(outcomes))
|
|
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
|
|
for index, outcome := range outcomes {
|
|
profile := ComparisonProfileResult{
|
|
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
|
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
|
|
}
|
|
if published && outcome.Status == comparison.StatusSucceeded {
|
|
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
|
|
}
|
|
result.Results[index] = profile
|
|
if outcome.Status == comparison.StatusSucceeded {
|
|
result.Succeeded++
|
|
} else {
|
|
result.Failed++
|
|
}
|
|
}
|
|
}
|
|
|
|
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
|
|
manifest := comparison.Manifest{
|
|
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
|
|
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
|
|
ReportID: string(result.ReportID), Timezone: result.Timezone,
|
|
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
|
|
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
|
|
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
|
|
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
|
|
Results: make([]comparison.Result, len(outcomes)),
|
|
}
|
|
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
|
|
for index, outcome := range outcomes {
|
|
manifestResult := comparison.Result{
|
|
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
|
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
|
|
}
|
|
if outcome.Status == comparison.StatusSucceeded {
|
|
manifestResult.ReportPath = outcome.ReportPath
|
|
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
|
|
}
|
|
bundle.Manifest.Results[index] = manifestResult
|
|
}
|
|
return bundle
|
|
}
|