Retain generation response evidence

docs/roadmap/implementation.md

@@ -439,6 +439,11 @@ Validate: focused `internal/llm` tests for valid/empty/malformed generation,
 non-success status, and a decoded response followed by cancellation. Reuse
 existing envelope/usage/size tests rather than reproduce their full matrices.
 
+**Completed: 2026-10-13.** Added an evidence-enabled prepared execution path
+while keeping ordinary execution on its existing HTTP path. Validation passed:
+`go test -race ./internal/llm`, `go vet ./internal/llm`,
+`go test ./...`, and `git diff --check`.
+
 ## Stage 9: Carry Evidence Through Built-In Retry Coordination
 
 Scope: `internal/generation` and shared observation usage accounting.

internal/llm/openai_compatible_client.go

@@ -101,24 +101,29 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
 }
 
 // PreparedGeneration holds one encoded request and resolved authentication for
-// repeated attempts. Each Execute call creates a fresh HTTP request and context.
+// repeated attempts. Each execution creates a fresh HTTP request and context.
 type PreparedGeneration struct {
-	endpoint       string
+	endpoint                  string
-	payload        []byte
+	payload                   []byte
-	apiKey         string
+	apiKey                    string
-	timeoutSeconds int
+	timeoutSeconds            int
-	httpClient     *http.Client
+	httpClient                *http.Client
+	structuredOutputRequested bool
 }
 
 // AttemptOutcome describes one HTTP execution without exposing response bodies
 // or headers to the orchestration layer.
 type AttemptOutcome struct {
-	Response       *domain.GenerateResponse
+	Response                  *domain.GenerateResponse
-	Err            error
+	Err                       error
-	StatusCode     int
+	StatusCode                int
-	Usage          domain.TokenUsage
+	Usage                     domain.TokenUsage
-	UsagePresence  *domain.TokenUsagePresence
+	UsagePresence             *domain.TokenUsagePresence
-	RetryNotBefore time.Time
+	RetryNotBefore            time.Time
+	Evidence                  HTTPExecutionEvidence
+	StructuredOutputRequested *bool
+	ResponseDecoded           bool
+	GenerationCompleted       bool
 }
 
 // PrepareGeneration validates and freezes one built-in request before dispatch.
@@ -177,12 +182,23 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
 	return &PreparedGeneration{
 		endpoint: endpoint, payload: payload, apiKey: apiKey,
 		timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
+		structuredOutputRequested: wireReq.ResponseFormat != nil,
 	}, nil
 }
 
 // Execute dispatches one attempt. The returned response is non-nil only when
 // the successful envelope contains a usable first choice.
 func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
+	return p.execute(ctx, false)
+}
+
+// ExecuteWithEvidence performs one prepared attempt and retains HTTP and
+// decoded-generation facts even when a later context check returns an error.
+func (p *PreparedGeneration) ExecuteWithEvidence(ctx context.Context) AttemptOutcome {
+	return p.execute(ctx, true)
+}
+
+func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) AttemptOutcome {
 	requestContext := ctx
 	if p.timeoutSeconds > 0 {
 		var cancel context.CancelFunc
@@ -198,13 +214,29 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
 		httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
 	}
 
-	httpResp, err := p.httpClient.Do(httpReq)
+	var (
+		httpResp *http.Response
+		outcome  AttemptOutcome
+	)
+	if captureEvidence {
+		httpResp, err, outcome.Evidence = doWithEvidence(p.httpClient, httpReq)
+		if outcome.Evidence.DispatchInvoked {
+			requested := p.structuredOutputRequested
+			outcome.StructuredOutputRequested = &requested
+		}
+	} else {
+		httpResp, err = p.httpClient.Do(httpReq)
+	}
+	if httpResp != nil && (captureEvidence || err == nil) {
+		outcome.StatusCode = httpResp.StatusCode
+	}
 	if err != nil {
-		return AttemptOutcome{Err: &requestFailedError{cause: err}}
+		outcome.Err = &requestFailedError{cause: err}
+		return outcome
 	}
 	receivedAt := time.Now()
 	defer httpResp.Body.Close()
-	outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
+	outcome.StatusCode = httpResp.StatusCode
 
 	if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
 		outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
@@ -220,6 +252,11 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
 	}
 	if httpResp.ContentLength > maxOpenAIChatResponseBytes {
 		outcome.Err = openAIChatResponseTooLargeError()
+		if captureEvidence {
+			if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
+				outcome.Err = contextErr
+			}
+		}
 		return outcome
 	}
 
@@ -231,6 +268,7 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
 		}
 		return outcome
 	}
+	outcome.ResponseDecoded = true
 	outcome.Usage = domain.TokenUsage{
 		PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
 		TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
@@ -244,14 +282,30 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
 
 	if len(wireResp.Choices) == 0 {
 		outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
+		if captureEvidence {
+			if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
+				outcome.Err = contextErr
+			}
+		}
 		return outcome
 	}
 	content := wireResp.Choices[0].Message.Content
 	if content == nil {
 		outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
+		if captureEvidence {
+			if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
+				outcome.Err = contextErr
+			}
+		}
 		return outcome
 	}
 	outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
+	outcome.GenerationCompleted = true
+	if captureEvidence {
+		if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
+			outcome.Err = contextErr
+		}
+	}
 	return outcome
 }
 

internal/llm/prepared_generation_evidence_test.go

@@ -0,0 +1,222 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testing.T) {
+	tests := []struct {
+		name               string
+		status             int
+		body               string
+		structured         bool
+		wantDecoded        bool
+		wantGeneration     bool
+		wantResponse       bool
+		wantContent        string
+		wantFailure        error
+		wantUsagePresence  bool
+		wantStructuredFact bool
+	}{
+		{
+			name:               "valid content",
+			status:             http.StatusOK,
+			body:               `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`,
+			structured:         true,
+			wantDecoded:        true,
+			wantGeneration:     true,
+			wantResponse:       true,
+			wantContent:        "ok",
+			wantUsagePresence:  true,
+			wantStructuredFact: true,
+		},
+		{
+			name:               "empty content",
+			status:             http.StatusOK,
+			body:               `{"choices":[{"message":{"content":""}}]}`,
+			wantDecoded:        true,
+			wantGeneration:     true,
+			wantResponse:       true,
+			wantStructuredFact: false,
+		},
+		{
+			name:               "decoded envelope without generated content",
+			status:             http.StatusOK,
+			body:               `{"choices":[],"usage":{"prompt_tokens":0}}`,
+			wantDecoded:        true,
+			wantUsagePresence:  true,
+			wantFailure:        ErrMalformedResponse,
+			wantStructuredFact: false,
+		},
+		{
+			name:               "malformed envelope",
+			status:             http.StatusOK,
+			body:               `not json`,
+			wantFailure:        ErrMalformedResponse,
+			wantStructuredFact: false,
+		},
+		{
+			name:               "provider status",
+			status:             http.StatusTooManyRequests,
+			body:               `{"error":{"message":"try again"}}`,
+			wantFailure:        ErrUnexpectedStatus,
+			wantStructuredFact: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
+				BaseURL: "https://provider.example/v1",
+				Model:   "model",
+				HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
+					response := responseFor(request, test.status, "")
+					response.Body = io.NopCloser(strings.NewReader(test.body))
+					response.ContentLength = int64(len(test.body))
+					return response, nil
+				})},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			request := ordinaryGenerateRequest()
+			if test.structured {
+				request.StructuredOutput = &domain.StructuredOutputSpec{
+					Type: domain.StructuredOutputJSONSchema,
+					JSONSchema: &domain.StructuredOutputJSONSpec{
+						Name:   "smoke",
+						Schema: map[string]any{"type": "object"},
+					},
+				}
+			}
+			prepared, err := client.PrepareGeneration(request)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			outcome := prepared.ExecuteWithEvidence(context.Background())
+			if (outcome.Err != nil) != (test.wantFailure != nil) || (test.wantFailure != nil && !errors.Is(outcome.Err, test.wantFailure)) {
+				t.Fatalf("error = %v, want failure %v", outcome.Err, test.wantFailure)
+			}
+			if !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved || !outcome.Evidence.ResponseReceived {
+				t.Fatalf("response evidence = %+v", outcome.Evidence)
+			}
+			if outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != test.status {
+				t.Fatalf("HTTP evidence status = %v, want %d", outcome.Evidence.HTTPStatus, test.status)
+			}
+			if outcome.GenerationCompleted != test.wantGeneration {
+				t.Fatalf("generation completed = %t, want %t", outcome.GenerationCompleted, test.wantGeneration)
+			}
+			if outcome.ResponseDecoded != test.wantDecoded {
+				t.Fatalf("response decoded = %t, want %t", outcome.ResponseDecoded, test.wantDecoded)
+			}
+			if (outcome.Response != nil) != test.wantResponse {
+				t.Fatalf("response presence = %t, want %t", outcome.Response != nil, test.wantResponse)
+			}
+			if test.wantResponse && outcome.Response.Content != test.wantContent {
+				t.Fatalf("generated content = %q, want %q", outcome.Response.Content, test.wantContent)
+			}
+			if (outcome.UsagePresence != nil && outcome.UsagePresence.PromptTokens) != test.wantUsagePresence {
+				t.Fatalf("usage presence = %+v, want prompt token presence %t", outcome.UsagePresence, test.wantUsagePresence)
+			}
+			if outcome.StructuredOutputRequested == nil || *outcome.StructuredOutputRequested != test.wantStructuredFact {
+				t.Fatalf("structured-output evidence = %v, want %t", outcome.StructuredOutputRequested, test.wantStructuredFact)
+			}
+		})
+	}
+}
+
+func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
+	client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
+		BaseURL: "https://provider.example/v1",
+		Model:   "model",
+		HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
+			return &http.Response{
+				StatusCode:    http.StatusOK,
+				Header:        make(http.Header),
+				Body:          &cancelAfterReadBody{Reader: strings.NewReader(body), cancel: cancel},
+				ContentLength: int64(len(body)),
+				Request:       request,
+			}, nil
+		})},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	outcome := prepared.ExecuteWithEvidence(ctx)
+	if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) {
+		t.Fatalf("canceled outcome = %+v", outcome)
+	}
+	if !outcome.Evidence.ResponseReceived || outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != http.StatusOK {
+		t.Fatalf("response evidence was lost after cancellation: %+v", outcome.Evidence)
+	}
+	if !outcome.ResponseDecoded || !outcome.GenerationCompleted || outcome.Response == nil || outcome.Response.Content != "ok" {
+		t.Fatalf("decoded generation was lost after cancellation: %+v", outcome)
+	}
+	if outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 {
+		t.Fatalf("decoded usage was lost after cancellation: %+v", outcome)
+	}
+}
+
+func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
+	transportErr := errors.New("provider unavailable")
+	client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
+		BaseURL: "https://provider.example/v1",
+		Model:   "model",
+		HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+			return nil, transportErr
+		})},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	request := ordinaryGenerateRequest()
+	request.StructuredOutput = &domain.StructuredOutputSpec{
+		Type: domain.StructuredOutputJSONSchema,
+		JSONSchema: &domain.StructuredOutputJSONSpec{
+			Name:   "smoke",
+			Schema: map[string]any{"type": "object"},
+		},
+	}
+	prepared, err := client.PrepareGeneration(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	outcome := prepared.ExecuteWithEvidence(context.Background())
+	if !errors.Is(outcome.Err, transportErr) || !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved {
+		t.Fatalf("dispatch outcome = %+v", outcome)
+	}
+	if outcome.Evidence.ResponseReceived || outcome.StructuredOutputRequested == nil || !*outcome.StructuredOutputRequested {
+		t.Fatalf("structured request evidence = %+v, request=%v", outcome.Evidence, outcome.StructuredOutputRequested)
+	}
+}
+
+type cancelAfterReadBody struct {
+	io.Reader
+	cancel context.CancelFunc
+}
+
+func (b *cancelAfterReadBody) Read(buffer []byte) (int, error) {
+	n, err := b.Reader.Read(buffer)
+	if n > 0 {
+		b.cancel()
+	}
+	return n, err
+}
+
+func (b *cancelAfterReadBody) Close() error { return nil }