eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retain generation response evidence

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-13 16:47:26 +00:00
parent 6eeb0ee9c0
commit d91afc61a4
3 changed files with 296 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/roadmap/implementation.md
View File

@@ -439,6 +439,11 @@ Validate: focused `internal/llm` tests for valid/empty/malformed generation,
non-success status, and a decoded response followed by cancellation. Reuse
existing envelope/usage/size tests rather than reproduce their full matrices.
**Completed: 2026-10-13.** Added an evidence-enabled prepared execution path
while keeping ordinary execution on its existing HTTP path. Validation passed:
`go test -race ./internal/llm`, `go vet ./internal/llm`,
`go test ./...`, and `git diff --check`.
## Stage 9: Carry Evidence Through Built-In Retry Coordination
Scope: `internal/generation` and shared observation usage accounting.

84
internal/llm/openai_compatible_client.go
View File

@@ -101,24 +101,29 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
}
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each Execute call creates a fresh HTTP request and context.
// repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct {
endpoint string
payload []byte
apiKey string
timeoutSeconds int
httpClient *http.Client
endpoint string
payload []byte
apiKey string
timeoutSeconds int
httpClient *http.Client
structuredOutputRequested bool
}
// AttemptOutcome describes one HTTP execution without exposing response bodies
// or headers to the orchestration layer.
type AttemptOutcome struct {
Response *domain.GenerateResponse
Err error
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
RetryNotBefore time.Time
Response *domain.GenerateResponse
Err error
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
RetryNotBefore time.Time
Evidence HTTPExecutionEvidence
StructuredOutputRequested *bool
ResponseDecoded bool
GenerationCompleted bool
}
// PrepareGeneration validates and freezes one built-in request before dispatch.
@@ -177,12 +182,23 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wireReq.ResponseFormat != nil,
}, nil
}
// Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
return p.execute(ctx, false)
}
// ExecuteWithEvidence performs one prepared attempt and retains HTTP and
// decoded-generation facts even when a later context check returns an error.
func (p *PreparedGeneration) ExecuteWithEvidence(ctx context.Context) AttemptOutcome {
return p.execute(ctx, true)
}
func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool) AttemptOutcome {
requestContext := ctx
if p.timeoutSeconds > 0 {
var cancel context.CancelFunc
@@ -198,13 +214,29 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
}
httpResp, err := p.httpClient.Do(httpReq)
var (
httpResp *http.Response
outcome AttemptOutcome
)
if captureEvidence {
httpResp, err, outcome.Evidence = doWithEvidence(p.httpClient, httpReq)
if outcome.Evidence.DispatchInvoked {
requested := p.structuredOutputRequested
outcome.StructuredOutputRequested = &requested
}
} else {
httpResp, err = p.httpClient.Do(httpReq)
}
if httpResp != nil && (captureEvidence || err == nil) {
outcome.StatusCode = httpResp.StatusCode
}
if err != nil {
return AttemptOutcome{Err: &requestFailedError{cause: err}}
outcome.Err = &requestFailedError{cause: err}
return outcome
}
receivedAt := time.Now()
defer httpResp.Body.Close()
outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
outcome.StatusCode = httpResp.StatusCode
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
@@ -220,6 +252,11 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
}
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
outcome.Err = openAIChatResponseTooLargeError()
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
@@ -231,6 +268,7 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
}
return outcome
}
outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
@@ -244,14 +282,30 @@ func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
if len(wireResp.Choices) == 0 {
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
content := wireResp.Choices[0].Message.Content
if content == nil {
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
outcome.GenerationCompleted = true
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
outcome.Err = contextErr
}
}
return outcome
}

222
internal/llm/prepared_generation_evidence_test.go Normal file
View File

@@ -0,0 +1,222 @@
package llm
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testing.T) {
tests := []struct {
name string
status int
body string
structured bool
wantDecoded bool
wantGeneration bool
wantResponse bool
wantContent string
wantFailure error
wantUsagePresence bool
wantStructuredFact bool
}{
{
name: "valid content",
status: http.StatusOK,
body: `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`,
structured: true,
wantDecoded: true,
wantGeneration: true,
wantResponse: true,
wantContent: "ok",
wantUsagePresence: true,
wantStructuredFact: true,
},
{
name: "empty content",
status: http.StatusOK,
body: `{"choices":[{"message":{"content":""}}]}`,
wantDecoded: true,
wantGeneration: true,
wantResponse: true,
wantStructuredFact: false,
},
{
name: "decoded envelope without generated content",
status: http.StatusOK,
body: `{"choices":[],"usage":{"prompt_tokens":0}}`,
wantDecoded: true,
wantUsagePresence: true,
wantFailure: ErrMalformedResponse,
wantStructuredFact: false,
},
{
name: "malformed envelope",
status: http.StatusOK,
body: `not json`,
wantFailure: ErrMalformedResponse,
wantStructuredFact: false,
},
{
name: "provider status",
status: http.StatusTooManyRequests,
body: `{"error":{"message":"try again"}}`,
wantFailure: ErrUnexpectedStatus,
wantStructuredFact: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
response := responseFor(request, test.status, "")
response.Body = io.NopCloser(strings.NewReader(test.body))
response.ContentLength = int64(len(test.body))
return response, nil
})},
})
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest()
if test.structured {
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "smoke",
Schema: map[string]any{"type": "object"},
},
}
}
prepared, err := client.PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(context.Background())
if (outcome.Err != nil) != (test.wantFailure != nil) || (test.wantFailure != nil && !errors.Is(outcome.Err, test.wantFailure)) {
t.Fatalf("error = %v, want failure %v", outcome.Err, test.wantFailure)
}
if !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved || !outcome.Evidence.ResponseReceived {
t.Fatalf("response evidence = %+v", outcome.Evidence)
}
if outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != test.status {
t.Fatalf("HTTP evidence status = %v, want %d", outcome.Evidence.HTTPStatus, test.status)
}
if outcome.GenerationCompleted != test.wantGeneration {
t.Fatalf("generation completed = %t, want %t", outcome.GenerationCompleted, test.wantGeneration)
}
if outcome.ResponseDecoded != test.wantDecoded {
t.Fatalf("response decoded = %t, want %t", outcome.ResponseDecoded, test.wantDecoded)
}
if (outcome.Response != nil) != test.wantResponse {
t.Fatalf("response presence = %t, want %t", outcome.Response != nil, test.wantResponse)
}
if test.wantResponse && outcome.Response.Content != test.wantContent {
t.Fatalf("generated content = %q, want %q", outcome.Response.Content, test.wantContent)
}
if (outcome.UsagePresence != nil && outcome.UsagePresence.PromptTokens) != test.wantUsagePresence {
t.Fatalf("usage presence = %+v, want prompt token presence %t", outcome.UsagePresence, test.wantUsagePresence)
}
if outcome.StructuredOutputRequested == nil || *outcome.StructuredOutputRequested != test.wantStructuredFact {
t.Fatalf("structured-output evidence = %v, want %t", outcome.StructuredOutputRequested, test.wantStructuredFact)
}
})
}
}
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: &cancelAfterReadBody{Reader: strings.NewReader(body), cancel: cancel},
ContentLength: int64(len(body)),
Request: request,
}, nil
})},
})
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(ctx)
if !errors.Is(outcome.Err, context.Canceled) || !errors.Is(outcome.Err, ErrTransport) {
t.Fatalf("canceled outcome = %+v", outcome)
}
if !outcome.Evidence.ResponseReceived || outcome.Evidence.HTTPStatus == nil || *outcome.Evidence.HTTPStatus != http.StatusOK {
t.Fatalf("response evidence was lost after cancellation: %+v", outcome.Evidence)
}
if !outcome.ResponseDecoded || !outcome.GenerationCompleted || outcome.Response == nil || outcome.Response.Content != "ok" {
t.Fatalf("decoded generation was lost after cancellation: %+v", outcome)
}
if outcome.UsagePresence == nil || !outcome.UsagePresence.PromptTokens || outcome.Usage.PromptTokens != 0 {
t.Fatalf("decoded usage was lost after cancellation: %+v", outcome)
}
}
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
transportErr := errors.New("provider unavailable")
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportErr
})},
})
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest()
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "smoke",
Schema: map[string]any{"type": "object"},
},
}
prepared, err := client.PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.ExecuteWithEvidence(context.Background())
if !errors.Is(outcome.Err, transportErr) || !outcome.Evidence.DispatchInvoked || !outcome.Evidence.RequestObserved {
t.Fatalf("dispatch outcome = %+v", outcome)
}
if outcome.Evidence.ResponseReceived || outcome.StructuredOutputRequested == nil || !*outcome.StructuredOutputRequested {
t.Fatalf("structured request evidence = %+v, request=%v", outcome.Evidence, outcome.StructuredOutputRequested)
}
}
type cancelAfterReadBody struct {
io.Reader
cancel context.CancelFunc
}
func (b *cancelAfterReadBody) Read(buffer []byte) (int, error) {
n, err := b.Reader.Read(buffer)
if n > 0 {
b.cancel()
}
return n, err
}
func (b *cancelAfterReadBody) Close() error { return nil }