diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index adb3c67..843d133 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -439,6 +439,11 @@ Validate: focused `internal/llm` tests for valid/empty/malformed generation, non-success status, and a decoded response followed by cancellation. Reuse existing envelope/usage/size tests rather than reproduce their full matrices. +**Completed: 2026-10-13.** Added an evidence-enabled prepared execution path +while keeping ordinary execution on its existing HTTP path. Validation passed: +`go test -race ./internal/llm`, `go vet ./internal/llm`, +`go test ./...`, and `git diff --check`. + ## Stage 9: Carry Evidence Through Built-In Retry Coordination Scope: `internal/generation` and shared observation usage accounting. diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index e32aba3..14e38a6 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -101,24 +101,29 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera } // PreparedGeneration holds one encoded request and resolved authentication for -// repeated attempts. Each Execute call creates a fresh HTTP request and context. +// repeated attempts. Each execution creates a fresh HTTP request and context. type PreparedGeneration struct { - endpoint string - payload []byte - apiKey string - timeoutSeconds int - httpClient *http.Client + endpoint string + payload []byte + apiKey string + timeoutSeconds int + httpClient *http.Client + structuredOutputRequested bool } // AttemptOutcome describes one HTTP execution without exposing response bodies // or headers to the orchestration layer. type AttemptOutcome struct { - Response *domain.GenerateResponse - Err error - StatusCode int - Usage domain.TokenUsage - UsagePresence *domain.TokenUsagePresence - RetryNotBefore time.Time + Response *domain.GenerateResponse + Err error + StatusCode int + Usage domain.TokenUsage + UsagePresence *domain.TokenUsagePresence + RetryNotBefore time.Time + Evidence HTTPExecutionEvidence + StructuredOutputRequested *bool + ResponseDecoded bool + GenerationCompleted bool } // PrepareGeneration validates and freezes one built-in request before dispatch. @@ -177,12 +182,23 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) ( return &PreparedGeneration{ endpoint: endpoint, payload: payload, apiKey: apiKey, timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient, + structuredOutputRequested: wireReq.ResponseFormat != nil, }, nil } // Execute dispatches one attempt. The returned response is non-nil only when // the successful envelope contains a usable first choice. func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { + return p.execute(ctx, false) +} + +// ExecuteWithEvidence performs one prepared attempt and retains HTTP and +// decoded-generation facts even when a later context check returns an error. +func (p *PreparedGeneration) ExecuteWithEvidence(ctx context.Context) AttemptOutcome { + return p.execute(ctx, true) +} + +func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) AttemptOutcome { requestContext := ctx if p.timeoutSeconds > 0 { var cancel context.CancelFunc @@ -198,13 +214,29 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { httpReq.Header.Set("Authorization", "Bearer "+p.apiKey) } - httpResp, err := p.httpClient.Do(httpReq) + var ( + httpResp *http.Response + outcome AttemptOutcome + ) + if captureEvidence { + httpResp, err, outcome.Evidence = doWithEvidence(p.httpClient, httpReq) + if outcome.Evidence.DispatchInvoked { + requested := p.structuredOutputRequested + outcome.StructuredOutputRequested = &requested + } + } else { + httpResp, err = p.httpClient.Do(httpReq) + } + if httpResp != nil && (captureEvidence || err == nil) { + outcome.StatusCode = httpResp.StatusCode + } if err != nil { - return AttemptOutcome{Err: &requestFailedError{cause: err}} + outcome.Err = &requestFailedError{cause: err} + return outcome } receivedAt := time.Now() defer httpResp.Body.Close() - outcome := AttemptOutcome{StatusCode: httpResp.StatusCode} + outcome.StatusCode = httpResp.StatusCode if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt) @@ -220,6 +252,11 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { } if httpResp.ContentLength > maxOpenAIChatResponseBytes { outcome.Err = openAIChatResponseTooLargeError() + if captureEvidence { + if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil { + outcome.Err = contextErr + } + } return outcome } @@ -231,6 +268,7 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { } return outcome } + outcome.ResponseDecoded = true outcome.Usage = domain.TokenUsage{ PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens), TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens), @@ -244,14 +282,30 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { if len(wireResp.Choices) == 0 { outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse) + if captureEvidence { + if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil { + outcome.Err = contextErr + } + } return outcome } content := wireResp.Choices[0].Message.Content if content == nil { outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse) + if captureEvidence { + if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil { + outcome.Err = contextErr + } + } return outcome } outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence} + outcome.GenerationCompleted = true + if captureEvidence { + if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil { + outcome.Err = contextErr + } + } return outcome } diff --git a/internal/llm/prepared_generation_evidence_test.go b/internal/llm/prepared_generation_evidence_test.go new file mode 100644 index 0000000..9245da7 --- /dev/null +++ b/internal/llm/prepared_generation_evidence_test.go @@ -0,0 +1,222 @@ +package llm + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testing.T) { + tests := []struct { + name string + status int + body string + structured bool + wantDecoded bool + wantGeneration bool + wantResponse bool + wantContent string + wantFailure error + wantUsagePresence bool + wantStructuredFact bool + }{ + { + name: "valid content", + status: http.StatusOK, + body: `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`, + structured: true, + wantDecoded: true, + wantGeneration: true, + wantResponse: true, + wantContent: "ok", + wantUsagePresence: true, + wantStructuredFact: true, + }, + { + name: "empty content", + status: http.StatusOK, + body: `{"choices":[{"message":{"content":""}}]}`, + wantDecoded: true, + wantGeneration: true, + wantResponse: true, + wantStructuredFact: false, + }, + { + name: "decoded envelope without generated content", + status: http.StatusOK, + body: `{"choices":[],"usage":{"prompt_tokens":0}}`, + wantDecoded: true, + wantUsagePresence: true, + wantFailure: ErrMalformedResponse, + wantStructuredFact: false, + }, + { + name: "malformed envelope", + status: http.StatusOK, + body: `not json`, + wantFailure: ErrMalformedResponse, + wantStructuredFact: false, + }, + { + name: "provider status", + status: http.StatusTooManyRequests, + body: `{"error":{"message":"try again"}}`, + wantFailure: ErrUnexpectedStatus, + wantStructuredFact: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "https://provider.example/v1", + Model: "model", + HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + response := responseFor(request, test.status, "") + response.Body = io.NopCloser(strings.NewReader(test.body)) + response.ContentLength = int64(len(test.body)) + return response, nil + })}, + }) + if err != nil { + t.Fatal(err) + } + request := ordinaryGenerateRequest() + if test.structured { + request.StructuredOutput = &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: "smoke", + Schema: map[string]any{"type": "object"}, + }, + } + } + prepared, err := client.PrepareGeneration(request) + if err != nil { + t.Fatal(err) + } + + outcome := prepared.ExecuteWithEvidence(context.Background()) + if (outcome.Err != nil) != (test.wantFailure != nil) || (test.wantFailure != nil && !errors.Is(outcome.Err, test.wantFailure)) { + t.Fatalf("error = %v, want failure %v", outcome.Err, test.wantFailure) + } + if !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved || !outcome.Evidence.ResponseReceived { + t.Fatalf("response evidence = %+v", outcome.Evidence) + } + if outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != test.status { + t.Fatalf("HTTP evidence status = %v, want %d", outcome.Evidence.HTTPStatus, test.status) + } + if outcome.GenerationCompleted != test.wantGeneration { + t.Fatalf("generation completed = %t, want %t", outcome.GenerationCompleted, test.wantGeneration) + } + if outcome.ResponseDecoded != test.wantDecoded { + t.Fatalf("response decoded = %t, want %t", outcome.ResponseDecoded, test.wantDecoded) + } + if (outcome.Response != nil) != test.wantResponse { + t.Fatalf("response presence = %t, want %t", outcome.Response != nil, test.wantResponse) + } + if test.wantResponse && outcome.Response.Content != test.wantContent { + t.Fatalf("generated content = %q, want %q", outcome.Response.Content, test.wantContent) + } + if (outcome.UsagePresence != nil && outcome.UsagePresence.PromptTokens) != test.wantUsagePresence { + t.Fatalf("usage presence = %+v, want prompt token presence %t", outcome.UsagePresence, test.wantUsagePresence) + } + if outcome.StructuredOutputRequested == nil || *outcome.StructuredOutputRequested != test.wantStructuredFact { + t.Fatalf("structured-output evidence = %v, want %t", outcome.StructuredOutputRequested, test.wantStructuredFact) + } + }) + } +} + +func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}` + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "https://provider.example/v1", + Model: "model", + HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: &cancelAfterReadBody{Reader: strings.NewReader(body), cancel: cancel}, + ContentLength: int64(len(body)), + Request: request, + }, nil + })}, + }) + if err != nil { + t.Fatal(err) + } + prepared, err := client.PrepareGeneration(ordinaryGenerateRequest()) + if err != nil { + t.Fatal(err) + } + + outcome := prepared.ExecuteWithEvidence(ctx) + if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) { + t.Fatalf("canceled outcome = %+v", outcome) + } + if !outcome.Evidence.ResponseReceived || outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != http.StatusOK { + t.Fatalf("response evidence was lost after cancellation: %+v", outcome.Evidence) + } + if !outcome.ResponseDecoded || !outcome.GenerationCompleted || outcome.Response == nil || outcome.Response.Content != "ok" { + t.Fatalf("decoded generation was lost after cancellation: %+v", outcome) + } + if outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 { + t.Fatalf("decoded usage was lost after cancellation: %+v", outcome) + } +} + +func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) { + transportErr := errors.New("provider unavailable") + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ + BaseURL: "https://provider.example/v1", + Model: "model", + HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, transportErr + })}, + }) + if err != nil { + t.Fatal(err) + } + request := ordinaryGenerateRequest() + request.StructuredOutput = &domain.StructuredOutputSpec{ + Type: domain.StructuredOutputJSONSchema, + JSONSchema: &domain.StructuredOutputJSONSpec{ + Name: "smoke", + Schema: map[string]any{"type": "object"}, + }, + } + prepared, err := client.PrepareGeneration(request) + if err != nil { + t.Fatal(err) + } + + outcome := prepared.ExecuteWithEvidence(context.Background()) + if !errors.Is(outcome.Err, transportErr) || !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved { + t.Fatalf("dispatch outcome = %+v", outcome) + } + if outcome.Evidence.ResponseReceived || outcome.StructuredOutputRequested == nil || !*outcome.StructuredOutputRequested { + t.Fatalf("structured request evidence = %+v, request=%v", outcome.Evidence, outcome.StructuredOutputRequested) + } +} + +type cancelAfterReadBody struct { + io.Reader + cancel context.CancelFunc +} + +func (b *cancelAfterReadBody) Read(buffer []byte) (int, error) { + n, err := b.Reader.Read(buffer) + if n > 0 { + b.cancel() + } + return n, err +} + +func (b *cancelAfterReadBody) Close() error { return nil }