package app import ( "context" "errors" "fmt" "sync" "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" ) type comparisonExecutionRequest struct { Prepared preparedReport Inspection ComparisonInspectionResult ComparisonID string DebugWriter *promptdebug.PromptDebugWriter Executor promptexec.Executor } type comparisonExecutionResult struct { Outcomes []comparisonProfileOutcome Canceled bool } type comparisonProfileOutcome struct { Position int ProfileID string BackendID string ModelName string Status string ValidationStatus promptexec.ValidationStatus ReportPath string Markdown []byte LLMDebugPath string Error *comparison.SafeError canceled bool } type comparisonProfileExecutionState uint8 const ( comparisonProfilePending comparisonProfileExecutionState = iota comparisonProfileRunning comparisonProfileComplete ) func executeComparisonProfiles(ctx context.Context, req comparisonExecutionRequest) comparisonExecutionResult { profiles := req.Inspection.Profiles result := comparisonExecutionResult{Outcomes: make([]comparisonProfileOutcome, len(profiles))} states := make([]comparisonProfileExecutionState, len(profiles)) for index, profile := range profiles { result.Outcomes[index] = comparisonProfileOutcome{ Position: index + 1, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, Status: comparison.StatusFailed, } } var waitGroup sync.WaitGroup for index, profile := range profiles { if err := ctx.Err(); err != nil { result.Canceled = true break } index, profile := index, profile states[index] = comparisonProfileRunning waitGroup.Add(1) go func() { defer waitGroup.Done() result.Outcomes[index] = executeComparisonProfile(ctx, req, index, profile) states[index] = comparisonProfileComplete }() } waitGroup.Wait() if err := ctx.Err(); err != nil { result.Canceled = true for index := range result.Outcomes { if states[index] != comparisonProfileComplete || result.Outcomes[index].canceled { markCanceledComparisonOutcome(&result.Outcomes[index], err) } } } return result } func executeComparisonProfile(ctx context.Context, req comparisonExecutionRequest, index int, profile ComparisonProfileInspection) comparisonProfileOutcome { position := index + 1 outcome := comparisonProfileOutcome{ Position: position, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, Status: comparison.StatusFailed, } debugRef := promptdebug.PromptDebugRef{ ReportID: req.Prepared.resolved.Definition.ID, ValidDate: req.Prepared.resolved.ValidPeriod.Start.Format("2006-01-02"), RunID: comparisonDebugRunID(req.ComparisonID, position, len(req.Inspection.Profiles), profile.ProfileID), } execution, markdown, err := executePreparedProfile(ctx, profileExecutionRequest{ Prepared: req.Prepared, Prompt: PromptInspectionResult{ PromptID: req.Inspection.PromptID, PromptVersion: req.Inspection.PromptVersion, PromptHash: req.Inspection.PromptHash, }, Profile: promptexec.ProfileInspection{ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName}, Executor: req.Executor, DebugWriter: req.DebugWriter, DebugRef: &debugRef, }) outcome.ProfileID, outcome.BackendID, outcome.ModelName = execution.ProfileID, execution.BackendID, execution.ModelName outcome.ValidationStatus = execution.ValidationStatus outcome.LLMDebugPath = execution.LLMDebugPath if err != nil { outcome.canceled = cancellationError(err) safe := comparisonSafeExecutionError(err) outcome.Error = &safe return outcome } reportPath, err := comparison.ReportFilename(position, len(req.Inspection.Profiles), profile.ProfileID) if err != nil { safe := comparison.NewSafeError("application", "derive comparison report filename failed") outcome.Error = &safe return outcome } outcome.Status = comparison.StatusSucceeded outcome.ReportPath = reportPath outcome.Markdown = append([]byte(nil), markdown...) return outcome } func comparisonDebugRunID(comparisonID string, position, profileCount int, profileID string) string { return fmt.Sprintf("%s_%0*d-%s", comparisonID, comparison.OrdinalWidth(profileCount), position, comparison.ProfileSlug(profileID)) } func markCanceledComparisonOutcome(outcome *comparisonProfileOutcome, err error) { outcome.Status = comparison.StatusFailed outcome.ValidationStatus = promptexec.ValidationSkipped outcome.ReportPath = "" outcome.Markdown = nil safe := comparisonSafeExecutionError(err) outcome.Error = &safe } func cancellationError(err error) bool { category := promptexec.CategoryOf(err) return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || category == promptexec.Canceled || category == promptexec.DeadlineExceeded } func comparisonSafeExecutionError(err error) comparison.SafeError { category := promptexec.CategoryOf(err) if category == "" { switch { case errors.Is(err, context.Canceled): category = promptexec.Canceled case errors.Is(err, context.DeadlineExceeded): category = promptexec.DeadlineExceeded } } if category == "" { return comparison.NewSafeError("application", comparisonExecutionMessage(err)) } return comparison.NewSafeError(string(category), comparisonExecutionMessage(err)) } func comparisonExecutionMessage(err error) string { if errors.Is(err, context.Canceled) { return "profile execution canceled" } if errors.Is(err, context.DeadlineExceeded) { return "profile execution deadline exceeded" } var execution *profileExecutionError if errors.As(err, &execution) { return comparison.TruncateErrorMessage(execution.operation + " failed") } return "profile execution failed" }