285 lines
12 KiB
Go
285 lines
12 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/app"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
|
)
|
|
|
|
const (
|
|
commandGenerate = "generate"
|
|
commandRun = "run"
|
|
commandCompare = "compare"
|
|
|
|
summaryStatusSucceeded = "succeeded"
|
|
summaryStatusFailed = "failed"
|
|
)
|
|
|
|
type generateSummary struct {
|
|
Command string `json:"command"`
|
|
ReportID report.ID `json:"reportId"`
|
|
ReportName string `json:"reportName"`
|
|
PromptID string `json:"promptId"`
|
|
RunID string `json:"runId"`
|
|
Status string `json:"status"`
|
|
GeneratedAt time.Time `json:"generatedAt"`
|
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
|
OutputPath string `json:"outputPath,omitempty"`
|
|
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
|
PromptVersion string `json:"promptVersion"`
|
|
Timezone string `json:"timezone"`
|
|
ProfileID string `json:"profileId,omitempty"`
|
|
BackendID string `json:"backendId,omitempty"`
|
|
ModelName string `json:"modelName,omitempty"`
|
|
SourceWarnings []weatherdata.SourceWarning `json:"sourceWarnings,omitempty"`
|
|
ValidationStatus string `json:"validationStatus,omitempty"`
|
|
Notification *generateNotificationSummary `json:"notification,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type generateNotificationSummary struct {
|
|
Status string `json:"status,omitempty"`
|
|
UploadStatus string `json:"uploadStatus,omitempty"`
|
|
StatusError string `json:"statusError,omitempty"`
|
|
RunID string `json:"runId,omitempty"`
|
|
PipelineID string `json:"pipelineId,omitempty"`
|
|
BundleID string `json:"bundleId,omitempty"`
|
|
IdempotencyKey string `json:"idempotencyKey,omitempty"`
|
|
AcceptedAt *time.Time `json:"acceptedAt,omitempty"`
|
|
StartedAt *time.Time `json:"startedAt,omitempty"`
|
|
FinishedAt *time.Time `json:"finishedAt,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type batchSummary struct {
|
|
Command string `json:"command"`
|
|
Batch app.BatchKind `json:"batch"`
|
|
Status string `json:"status"`
|
|
StartedAt time.Time `json:"startedAt"`
|
|
FinishedAt time.Time `json:"finishedAt"`
|
|
Total int `json:"total"`
|
|
Succeeded int `json:"succeeded"`
|
|
Failed int `json:"failed"`
|
|
Canceled int `json:"canceled,omitempty"`
|
|
Notification *app.BatchNotificationResult `json:"notification,omitempty"`
|
|
Reports []app.BatchReportResult `json:"reports"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type comparisonSummary struct {
|
|
Command string `json:"command"`
|
|
ComparisonID string `json:"comparisonId"`
|
|
ReportID report.ID `json:"reportId"`
|
|
ReportName string `json:"reportName"`
|
|
PromptID string `json:"promptId"`
|
|
PromptVersion string `json:"promptVersion"`
|
|
PromptHash string `json:"promptHash"`
|
|
Status string `json:"status"`
|
|
StartedAt time.Time `json:"startedAt"`
|
|
FinishedAt time.Time `json:"finishedAt"`
|
|
Timezone string `json:"timezone"`
|
|
ValidPeriod timeutil.Period `json:"validPeriod"`
|
|
OutputDirectory string `json:"outputDirectory"`
|
|
ManifestPath string `json:"manifestPath,omitempty"`
|
|
DataPackagePath string `json:"dataPackagePath,omitempty"`
|
|
Total int `json:"total"`
|
|
Succeeded int `json:"succeeded"`
|
|
Failed int `json:"failed"`
|
|
Results []comparisonProfileSummary `json:"results"`
|
|
Error *comparison.SafeError `json:"error,omitempty"`
|
|
}
|
|
|
|
type comparisonProfileSummary struct {
|
|
Position int `json:"position"`
|
|
ProfileID string `json:"profileId"`
|
|
BackendID string `json:"backendId,omitempty"`
|
|
ModelName string `json:"modelName"`
|
|
Status string `json:"status"`
|
|
ValidationStatus string `json:"validationStatus,omitempty"`
|
|
ReportPath string `json:"reportPath,omitempty"`
|
|
LLMDebugPath string `json:"llmDebugPath,omitempty"`
|
|
Error *comparison.SafeError `json:"error,omitempty"`
|
|
}
|
|
|
|
func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
|
summary := generateSummary{Command: commandGenerate}
|
|
if result == nil {
|
|
return summary
|
|
}
|
|
|
|
summary.ReportID = result.ReportID
|
|
summary.ReportName = result.ReportName
|
|
summary.PromptID = result.PromptID
|
|
summary.PromptVersion = result.PromptVersion
|
|
summary.RunID = result.RunID
|
|
summary.Status = summaryStatusSucceeded
|
|
summary.GeneratedAt = result.GeneratedAt
|
|
summary.ValidPeriod = result.ValidPeriod
|
|
summary.Timezone = result.Timezone
|
|
summary.ProfileID, summary.BackendID, summary.ModelName = result.ProfileID, result.BackendID, result.ModelName
|
|
summary.SourceWarnings = append([]weatherdata.SourceWarning(nil), result.SourceWarnings...)
|
|
summary.ValidationStatus = string(result.ValidationStatus)
|
|
summary.OutputPath = result.OutputPath
|
|
summary.LLMDebugPath = result.LLMDebugPath
|
|
summary.Notification = newGenerateNotificationSummary(result.Notification)
|
|
if err != nil {
|
|
summary.Status = summaryStatusFailed
|
|
summary.Error = err.Error()
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func newGenerateNotificationSummary(result *app.NotificationResult) *generateNotificationSummary {
|
|
if result == nil {
|
|
return nil
|
|
}
|
|
summary := &generateNotificationSummary{
|
|
Status: result.Status,
|
|
UploadStatus: result.UploadStatus,
|
|
StatusError: result.StatusError,
|
|
RunID: result.RunID,
|
|
PipelineID: result.PipelineID,
|
|
BundleID: result.BundleID,
|
|
IdempotencyKey: result.IdempotencyKey,
|
|
StartedAt: result.StartedAt,
|
|
FinishedAt: result.FinishedAt,
|
|
Error: result.Error,
|
|
}
|
|
if !result.AcceptedAt.IsZero() {
|
|
acceptedAt := result.AcceptedAt
|
|
summary.AcceptedAt = &acceptedAt
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func newBatchSummary(result *app.BatchResult, err error) batchSummary {
|
|
summary := batchSummary{Command: commandRun}
|
|
if result == nil {
|
|
return summary
|
|
}
|
|
|
|
summary.Batch = result.Batch
|
|
summary.Status = batchSummaryStatus(result)
|
|
summary.StartedAt = result.StartedAt
|
|
summary.FinishedAt = result.FinishedAt
|
|
summary.Total = result.Total
|
|
summary.Succeeded = result.Succeeded
|
|
summary.Failed = result.Failed
|
|
summary.Canceled = result.Canceled
|
|
summary.Notification = result.Notification
|
|
summary.Reports = append([]app.BatchReportResult(nil), result.Reports...)
|
|
if summary.Status == summaryStatusFailed {
|
|
summary.Error = app.BatchError{Result: result, Cause: err}.Error()
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func newComparisonSummary(result *app.ComparisonResult, err error) comparisonSummary {
|
|
summary := comparisonSummary{Command: commandCompare, Results: []comparisonProfileSummary{}}
|
|
if result == nil {
|
|
return summary
|
|
}
|
|
summary.ComparisonID = result.ComparisonID
|
|
summary.ReportID, summary.ReportName = result.ReportID, result.ReportName
|
|
summary.PromptID, summary.PromptVersion, summary.PromptHash = result.PromptID, result.PromptVersion, result.PromptHash
|
|
summary.StartedAt, summary.FinishedAt = result.StartedAt, result.FinishedAt
|
|
summary.Timezone, summary.ValidPeriod = result.Timezone, result.ValidPeriod
|
|
summary.OutputDirectory = result.OutputDirectory
|
|
summary.ManifestPath, summary.DataPackagePath = result.ManifestPath, result.DataPackagePath
|
|
summary.Total, summary.Succeeded, summary.Failed = result.Total, result.Succeeded, result.Failed
|
|
for _, profile := range result.Results {
|
|
summary.Results = append(summary.Results, comparisonProfileSummary{
|
|
Position: profile.Position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
|
Status: profile.Status, ValidationStatus: string(profile.ValidationStatus), ReportPath: profile.ReportPath,
|
|
LLMDebugPath: profile.LLMDebugPath, Error: profile.Error,
|
|
})
|
|
}
|
|
summary.Status = comparisonSummaryStatus(result, err)
|
|
if err != nil {
|
|
summary.Error = safeComparisonSummaryError(err)
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func comparisonSummaryStatus(result *app.ComparisonResult, err error) string {
|
|
if result == nil || err != nil || result.Total < 2 || result.Succeeded != result.Total || result.Failed != 0 || result.ManifestPath == "" || result.DataPackagePath == "" {
|
|
return summaryStatusFailed
|
|
}
|
|
return summaryStatusSucceeded
|
|
}
|
|
|
|
func safeComparisonSummaryError(err error) *comparison.SafeError {
|
|
if aggregate, ok := comparisonAggregateErrorMessage(err); ok {
|
|
safe := comparison.NewSafeError("application", aggregate)
|
|
return &safe
|
|
}
|
|
var cleanupErr *comparison.PublicationCleanupError
|
|
if errors.As(err, &cleanupErr) {
|
|
safe := comparison.NewSafeError("publication_cleanup", "comparison published but cleanup did not complete")
|
|
return &safe
|
|
}
|
|
var destinationErr *comparison.DestinationError
|
|
if errors.As(err, &destinationErr) {
|
|
safe := comparison.NewSafeError("destination_"+string(destinationErr.Kind), "comparison destination preflight failed")
|
|
return &safe
|
|
}
|
|
if category := promptexec.CategoryOf(err); category != "" {
|
|
safe := comparison.NewSafeError(string(category), "comparison prompt operation failed")
|
|
return &safe
|
|
}
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
safe := comparison.NewSafeError("deadline_exceeded", "comparison deadline exceeded")
|
|
return &safe
|
|
}
|
|
if errors.Is(err, context.Canceled) {
|
|
safe := comparison.NewSafeError("canceled", "comparison canceled")
|
|
return &safe
|
|
}
|
|
safe := comparison.NewSafeError("application", "comparison did not complete")
|
|
return &safe
|
|
}
|
|
|
|
func comparisonAggregateErrorMessage(err error) (string, bool) {
|
|
const prefix = "comparison completed with "
|
|
const suffix = " failed profiles"
|
|
for candidate := err; candidate != nil; candidate = errors.Unwrap(candidate) {
|
|
value := candidate.Error()
|
|
if !strings.HasPrefix(value, prefix) || !strings.HasSuffix(value, suffix) {
|
|
continue
|
|
}
|
|
count, parseErr := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(value, prefix), suffix))
|
|
if parseErr == nil && count > 0 {
|
|
return fmt.Sprintf("comparison completed with %d failed profiles", count), true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func batchSummaryStatus(result *app.BatchResult) string {
|
|
if result == nil {
|
|
return ""
|
|
}
|
|
if result.Failed > 0 || result.Canceled > 0 || (result.Notification != nil && result.Notification.Status == summaryStatusFailed) {
|
|
return summaryStatusFailed
|
|
}
|
|
return summaryStatusSucceeded
|
|
}
|
|
|
|
func reportName(id report.ID) string {
|
|
definition, err := report.DefaultRegistry().Lookup(id)
|
|
if err != nil {
|
|
return string(id)
|
|
}
|
|
return definition.Name
|
|
}
|