Harden PromptKit upgrade integration

internal/app/comparison_execution_test.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -72,6 +74,66 @@ func TestExecuteComparisonProfilesContinuesAfterProfileFailure(t *testing.T) {
 	}
 }
 
+func TestExecuteComparisonProfilesPreservesIndependentRepairOutcomes(t *testing.T) {
+	prepared, prompt := preparedDailyProfile(t)
+	profiles := comparisonProfiles(4)
+	executor := newBarrierExecutor(profiles)
+	executor.setValidation(profiles[0].ProfileID, promptexec.ValidationPassed, 0)
+	executor.setValidation(profiles[1].ProfileID, promptexec.ValidationPassed, 1)
+	executor.setValidation(profiles[2].ProfileID, promptexec.ValidationFailed, 1)
+	executor.setError(profiles[3].ProfileID, errors.New("provider failure"))
+	results := startComparisonExecution(t, context.Background(), comparisonExecutionRequest{
+		Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", Executor: executor,
+	}, executor)
+	waitForProfileStarts(t, executor, profiles, results)
+	executor.releaseAll()
+	result := <-results
+	wantStatuses := []string{comparison.StatusSucceeded, comparison.StatusSucceeded, comparison.StatusFailed, comparison.StatusFailed}
+	wantValidations := []promptexec.ValidationStatus{promptexec.ValidationPassed, promptexec.ValidationPassed, promptexec.ValidationFailed, ""}
+	wantRepairs := []*int{intPointer(0), intPointer(1), intPointer(1), nil}
+	for index, outcome := range result.Outcomes {
+		if outcome.Status != wantStatuses[index] || outcome.ValidationStatus != wantValidations[index] || !reflect.DeepEqual(outcome.RepairAttempts, wantRepairs[index]) {
+			t.Fatalf("outcome[%d] = %#v, want status/validation/repairs %q/%q/%#v", index, outcome, wantStatuses[index], wantValidations[index], wantRepairs[index])
+		}
+	}
+}
+
+func TestExecuteComparisonProfilesCapturesConcurrentProviderFailures(t *testing.T) {
+	prepared, prompt := preparedDailyProfile(t)
+	profiles := comparisonProfiles(2)
+	debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
+	if errors.Is(err, promptdebug.ErrSecureCaptureUnsupported) {
+		t.Skipf("secure prompt debug capture is unavailable: %v", err)
+	}
+	if err != nil {
+		t.Fatalf("NewPromptDebugWriter() error = %v", err)
+	}
+	markers := []string{"first-provider-private-marker", "second-provider-private-marker"}
+	statuses := []int{http.StatusTooManyRequests, http.StatusServiceUnavailable}
+	executor := newBarrierExecutor(profiles)
+	for index, profile := range profiles {
+		executor.setError(profile.ProfileID, promptexec.NewGenerationError(statuses[index], "provider_code", "provider_type", markers[index], nil))
+	}
+	results := startComparisonExecution(t, context.Background(), comparisonExecutionRequest{
+		Prepared: prepared, Inspection: comparisonInspection(prompt, profiles), ComparisonID: "comparison_daily", DebugWriter: debugWriter, Executor: executor,
+	}, executor)
+	waitForProfileStarts(t, executor, profiles, results)
+	executor.releaseAll()
+	result := <-results
+	for index, outcome := range result.Outcomes {
+		if outcome.Status != comparison.StatusFailed || outcome.Error == nil || outcome.Error.Category != string(promptexec.Generation) || outcome.Error.Message != fmt.Sprintf("execute prompt failed (HTTP %d)", statuses[index]) || strings.Contains(outcome.Error.Message, markers[index]) || outcome.LLMDebugPath == "" {
+			t.Fatalf("outcome[%d] = %#v", index, outcome)
+		}
+		failure, readErr := os.ReadFile(filepath.Join(outcome.LLMDebugPath, "failure.json"))
+		if readErr != nil {
+			t.Fatal(readErr)
+		}
+		if !strings.Contains(string(failure), markers[index]) || strings.Contains(string(failure), markers[1-index]) {
+			t.Fatalf("failure[%d] = %s", index, failure)
+		}
+	}
+}
+
 func TestExecuteComparisonProfilesPropagatesCancellationAndJoins(t *testing.T) {
 	prepared, prompt := preparedDailyProfile(t)
 	profiles := comparisonProfiles(4)
@@ -139,6 +201,8 @@ type barrierExecutor struct {
 	releases         map[string]chan struct{}
 	requests         map[string]promptexec.ExecuteRequest
 	errors           map[string]error
+	validations      map[string]promptexec.ValidationStatus
+	repairAttempts   map[string]int
 	profiles         map[string]ComparisonProfileInspection
 	inFlight         int
 	maximum          int
@@ -153,7 +217,7 @@ func newBarrierExecutor(profiles []ComparisonProfileInspection) *barrierExecutor
 	}
 	return &barrierExecutor{
 		started: make(chan string, len(profiles)), callbackFailures: make(chan error, len(profiles)), releases: releases,
-		requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{}, profiles: identities,
+		requests: make(map[string]promptexec.ExecuteRequest, len(profiles)), errors: map[string]error{}, validations: map[string]promptexec.ValidationStatus{}, repairAttempts: map[string]int{}, profiles: identities,
 	}
 }
 
@@ -195,15 +259,20 @@ func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteReq
 	e.mu.Lock()
 	e.inFlight--
 	err := e.errors[req.ProfileID]
+	validationStatus := e.validations[req.ProfileID]
+	repairAttempts := e.repairAttempts[req.ProfileID]
 	e.mu.Unlock()
 	if err != nil {
 		return nil, err
 	}
+	if validationStatus == "" {
+		validationStatus = promptexec.ValidationPassed
+	}
 	return &promptexec.Execution{
 		PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash,
 		ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
 		StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
-		Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil),
+		Validation: promptexec.NewValidation(validationStatus, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", repairAttempts, nil),
 	}, nil
 }
 
@@ -220,6 +289,13 @@ func (e *barrierExecutor) setError(profileID string, err error) {
 	e.errors[profileID] = err
 }
 
+func (e *barrierExecutor) setValidation(profileID string, status promptexec.ValidationStatus, repairAttempts int) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	e.validations[profileID] = status
+	e.repairAttempts[profileID] = repairAttempts
+}
+
 func (e *barrierExecutor) release(profileID string) {
 	close(e.releases[profileID])
 }
@@ -318,4 +394,8 @@ func bytesEqual(left, right []byte) bool {
 	return reflect.DeepEqual(left, right)
 }
 
+func intPointer(value int) *int {
+	return &value
+}
+
 var _ promptexec.Executor = (*barrierExecutor)(nil)