Retain injected client diagnostic evidence

internal/generation/evidence_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
+	"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
 	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/llm"
 )
 
 func TestGenerateWithEvidenceRetainsAttemptsAndUsage(t *testing.T) {
@@ -173,6 +175,137 @@ func TestGenerateWithEvidenceNoDispatchHasCompleteZeroUsage(t *testing.T) {
 	}
 }
 
+func TestGenerateWithEvidenceForInjectedClient(t *testing.T) {
+	opaqueFailure := errors.New("injected client failed")
+	presence := &domain.TokenUsagePresence{PromptTokens: true}
+	for _, tc := range []struct {
+		name           string
+		response       *domain.GenerateResponse
+		err            error
+		wantResponse   bool
+		wantGeneration bool
+		wantFailure    domain.ExecutionFailureKind
+		wantUsage      bool
+		wantStructured bool
+	}{
+		{
+			name:           "success",
+			response:       &domain.GenerateResponse{Content: "private output", Usage: domain.TokenUsage{PromptTokens: 4}, UsagePresence: presence},
+			wantResponse:   true,
+			wantGeneration: true,
+			wantUsage:      true,
+			wantStructured: true,
+		},
+		{
+			name:        "nil response",
+			wantFailure: domain.ExecutionFailureMalformedResponse,
+		},
+		{
+			name:        "opaque failure",
+			err:         opaqueFailure,
+			wantFailure: domain.ExecutionFailureGeneration,
+		},
+		{
+			name:           "response with error",
+			response:       &domain.GenerateResponse{Content: "must not be evaluated", Usage: domain.TokenUsage{PromptTokens: 9}, UsagePresence: presence},
+			err:            opaqueFailure,
+			wantResponse:   true,
+			wantFailure:    domain.ExecutionFailureGeneration,
+			wantStructured: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			client := &evidenceInjectedClient{response: tc.response, err: tc.err}
+			coordinator := newEvidenceInjectedCoordinator(t, client)
+			req := testRequest(7)
+			if tc.wantStructured {
+				req.StructuredOutput = &domain.StructuredOutputSpec{}
+			}
+
+			result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), req)
+			if client.calls != 1 || len(evidence.Attempts) != 1 || len(evidence.RetryDelays) != 0 {
+				t.Fatalf("calls=%d attempts=%+v retry delays=%v", client.calls, evidence.Attempts, evidence.RetryDelays)
+			}
+			attempt := evidence.Attempts[0]
+			if !attempt.Dispatched || attempt.ResponseReceived != tc.wantResponse || attempt.GenerationCompleted != tc.wantGeneration || attempt.Failure != tc.wantFailure {
+				t.Fatalf("attempt evidence = %+v", attempt)
+			}
+			if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
+				t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
+			}
+			if attempt.StructuredOutputRequested == nil || *attempt.StructuredOutputRequested != tc.wantStructured {
+				t.Fatalf("structured-output request fact = %v", attempt.StructuredOutputRequested)
+			}
+			if tc.wantFailure == domain.ExecutionFailureNone {
+				if err != nil || result == nil || result.Content != "private output" {
+					t.Fatalf("result=%+v error=%v", result, err)
+				}
+			} else if result != nil || err == nil {
+				t.Fatalf("result=%+v error=%v; expected failed call", result, err)
+			}
+			if attempt.Usage.PromptTokens.Known != tc.wantUsage || evidence.Usage.PromptTokens.Known != tc.wantUsage {
+				t.Fatalf("attempt/aggregate usage = %+v / %+v", attempt.Usage, evidence.Usage)
+			}
+			if tc.wantUsage && (attempt.Usage.PromptTokens.Value != 4 || evidence.Usage.PromptTokens.Value != 4) {
+				t.Fatalf("reported usage was not retained: %+v / %+v", attempt.Usage, evidence.Usage)
+			}
+		})
+	}
+}
+
+func TestGenerateWithEvidenceRetainsInjectedSuccessAfterCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	client := &evidenceInjectedClient{
+		response:      &domain.GenerateResponse{Content: "private output"},
+		afterGenerate: func() { cancel() },
+	}
+	coordinator := newEvidenceInjectedCoordinator(t, client)
+	req := testRequest(5)
+	req.StructuredOutput = &domain.StructuredOutputSpec{}
+
+	result, err, evidence := coordinator.GenerateWithEvidence(ctx, req)
+	if !errors.Is(err, context.Canceled) || result == nil || result.Content != "private output" {
+		t.Fatalf("result=%+v error=%v", result, err)
+	}
+	if client.calls != 1 || len(evidence.Attempts) != 1 {
+		t.Fatalf("calls=%d attempts=%+v", client.calls, evidence.Attempts)
+	}
+	attempt := evidence.Attempts[0]
+	if !attempt.ResponseReceived || !attempt.GenerationCompleted || attempt.Failure != domain.ExecutionFailureNone {
+		t.Fatalf("successful generation evidence was lost: %+v", attempt)
+	}
+	if attempt.HTTP != (llm.HTTPExecutionEvidence{}) {
+		t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP)
+	}
+}
+
+type evidenceInjectedClient struct {
+	calls         int
+	response      *domain.GenerateResponse
+	err           error
+	afterGenerate func()
+}
+
+func (c *evidenceInjectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	c.calls++
+	if c.afterGenerate != nil {
+		c.afterGenerate()
+	}
+	return c.response, c.err
+}
+
+func newEvidenceInjectedCoordinator(t *testing.T, client llm.Client) *Coordinator {
+	t.Helper()
+	manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
+		"limited": {ConcurrencyLimit: 1},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return NewInjected(manager, client)
+}
+
 type cancelOnDoneContext struct {
 	context.Context
 	cancel context.CancelFunc