From 2b723760d118963131be63e386182afaba8e2147 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 13 Oct 2026 17:05:40 +0000 Subject: [PATCH] Retain injected client diagnostic evidence --- docs/roadmap/implementation.md | 6 ++ internal/generation/coordinator.go | 65 +++++++++++-- internal/generation/evidence.go | 1 + internal/generation/evidence_test.go | 133 +++++++++++++++++++++++++++ llm_adapter.go | 9 +- llm_adapter_internal_test.go | 31 +++++++ 6 files changed, 233 insertions(+), 12 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index bf7ca2c..e6dbd9f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -489,6 +489,12 @@ response-plus-error, and success followed by caller cancellation. Verify one dispatch despite a positive retry allowance and ownership at the existing client adapter boundary. Do not add methods to public `LLMClient`. +**Completed: 2026-10-13.** Injected generation now records one-call response, +completion, usage, and native-schema-request evidence without HTTP claims, while +ordinary generation behavior remains unchanged. Validation passed: +`go test -race ./internal/generation .`, `go vet ./...`, `go test ./...`, and +`git diff --check`. + ## Stage 11: Implement The Reachability Probe Scope: internal HTTP probe collaborator and focused use-case support decisions. diff --git a/internal/generation/coordinator.go b/internal/generation/coordinator.go index ebb56d4..2d0b8ee 100644 --- a/internal/generation/coordinator.go +++ b/internal/generation/coordinator.go @@ -91,8 +91,7 @@ func (c *Coordinator) generate( captureEvidence bool, ) (*domain.GenerateResponse, error, GenerationEvidence) { if c.injected != nil { - response, err := c.generateInjected(ctx, req) - return response, err, GenerationEvidence{} + return c.generateInjected(ctx, req, captureEvidence) } prepared, err := c.builtIn.PrepareGeneration(req) if err != nil { @@ -125,6 +124,7 @@ func (c *Coordinator) generate( if captureEvidence { attempt := AttemptEvidence{ Dispatched: dispatched, + ResponseReceived: outcome.Evidence.ResponseReceived, HTTP: cloneHTTPEvidence(outcome.Evidence), CapacityWait: capacityWait, RequestDuration: generationDuration, @@ -196,7 +196,19 @@ func (c *Coordinator) generate( } } -func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { +func (c *Coordinator) generateInjected( + ctx context.Context, + req domain.GenerateRequest, + captureEvidence bool, +) (*domain.GenerateResponse, error, GenerationEvidence) { + evidence := GenerationEvidence{} + var usage observation.UsageAccumulator + finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) { + if captureEvidence { + evidence.Usage = usage.Observed() + } + return response, err, evidence + } recorder := observation.FromContext(ctx) generation := req.Generation if generation == 0 { @@ -206,18 +218,30 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR release, err := c.manager.Acquire(ctx, req.Target.BackendID) capacityWait := c.capacityWait(req.Target.BackendID, start) if err != nil { + if captureEvidence { + evidence.Attempts = append(evidence.Attempts, AttemptEvidence{ + CapacityWait: capacityWait, + Failure: ClassifyFailure(err), + }) + } if recorder != nil { recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)}) } - return nil, err + return finish(nil, err) } defer release() if err := ctx.Err(); err != nil { release() + if captureEvidence { + evidence.Attempts = append(evidence.Attempts, AttemptEvidence{ + CapacityWait: capacityWait, + Failure: ClassifyFailure(err), + }) + } if recorder != nil { recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)}) } - return nil, err + return finish(nil, err) } dispatchStart := c.now() response, err := c.injected.Generate(ctx, req) @@ -225,6 +249,28 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR if err == nil && response == nil { err = llm.ErrMalformedResponse } + if captureEvidence { + attempt := AttemptEvidence{ + Dispatched: true, + ResponseReceived: response != nil, + RequestDuration: duration, + Failure: ClassifyFailure(err), + GenerationCompleted: err == nil && response != nil, + } + requested := req.StructuredOutput != nil + attempt.StructuredOutputRequested = &requested + var reportedUsage domain.TokenUsage + var reportedPresence *domain.TokenUsagePresence + if err == nil && response != nil { + reportedUsage = response.Usage + reportedPresence = response.UsagePresence + } + usage.Add(reportedUsage, reportedPresence) + var attemptUsage observation.UsageAccumulator + attemptUsage.Add(reportedUsage, reportedPresence) + attempt.Usage = attemptUsage.Observed() + evidence.Attempts = append(evidence.Attempts, attempt) + } release() if recorder != nil { data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration, @@ -236,12 +282,15 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR recorder.Attempt(data) } if err != nil { - return nil, err + return finish(nil, err) } if err := ctx.Err(); err != nil { - return nil, err + if captureEvidence { + return finish(response, err) + } + return finish(nil, err) } - return response, nil + return finish(response, nil) } func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) { diff --git a/internal/generation/evidence.go b/internal/generation/evidence.go index 7685e6d..134237c 100644 --- a/internal/generation/evidence.go +++ b/internal/generation/evidence.go @@ -19,6 +19,7 @@ type GenerationEvidence struct { // reached. Generated content remains on the separate response return value. type AttemptEvidence struct { Dispatched bool + ResponseReceived bool HTTP llm.HTTPExecutionEvidence CapacityWait time.Duration RequestDuration time.Duration diff --git a/internal/generation/evidence_test.go b/internal/generation/evidence_test.go index a658ba0..d210073 100644 --- a/internal/generation/evidence_test.go +++ b/internal/generation/evidence_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "gitea.maximumdirect.net/eric/promptkit/internal/capacity" "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/llm" ) func TestGenerateWithEvidenceRetainsAttemptsAndUsage(t *testing.T) { @@ -173,6 +175,137 @@ func TestGenerateWithEvidenceNoDispatchHasCompleteZeroUsage(t *testing.T) { } } +func TestGenerateWithEvidenceForInjectedClient(t *testing.T) { + opaqueFailure := errors.New("injected client failed") + presence := &domain.TokenUsagePresence{PromptTokens: true} + for _, tc := range []struct { + name string + response *domain.GenerateResponse + err error + wantResponse bool + wantGeneration bool + wantFailure domain.ExecutionFailureKind + wantUsage bool + wantStructured bool + }{ + { + name: "success", + response: &domain.GenerateResponse{Content: "private output", Usage: domain.TokenUsage{PromptTokens: 4}, UsagePresence: presence}, + wantResponse: true, + wantGeneration: true, + wantUsage: true, + wantStructured: true, + }, + { + name: "nil response", + wantFailure: domain.ExecutionFailureMalformedResponse, + }, + { + name: "opaque failure", + err: opaqueFailure, + wantFailure: domain.ExecutionFailureGeneration, + }, + { + name: "response with error", + response: &domain.GenerateResponse{Content: "must not be evaluated", Usage: domain.TokenUsage{PromptTokens: 9}, UsagePresence: presence}, + err: opaqueFailure, + wantResponse: true, + wantFailure: domain.ExecutionFailureGeneration, + wantStructured: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + client := &evidenceInjectedClient{response: tc.response, err: tc.err} + coordinator := newEvidenceInjectedCoordinator(t, client) + req := testRequest(7) + if tc.wantStructured { + req.StructuredOutput = &domain.StructuredOutputSpec{} + } + + result, err, evidence := coordinator.GenerateWithEvidence(context.Background(), req) + if client.calls != 1 || len(evidence.Attempts) != 1 || len(evidence.RetryDelays) != 0 { + t.Fatalf("calls=%d attempts=%+v retry delays=%v", client.calls, evidence.Attempts, evidence.RetryDelays) + } + attempt := evidence.Attempts[0] + if !attempt.Dispatched || attempt.ResponseReceived != tc.wantResponse || attempt.GenerationCompleted != tc.wantGeneration || attempt.Failure != tc.wantFailure { + t.Fatalf("attempt evidence = %+v", attempt) + } + if attempt.HTTP != (llm.HTTPExecutionEvidence{}) { + t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP) + } + if attempt.StructuredOutputRequested == nil || *attempt.StructuredOutputRequested != tc.wantStructured { + t.Fatalf("structured-output request fact = %v", attempt.StructuredOutputRequested) + } + if tc.wantFailure == domain.ExecutionFailureNone { + if err != nil || result == nil || result.Content != "private output" { + t.Fatalf("result=%+v error=%v", result, err) + } + } else if result != nil || err == nil { + t.Fatalf("result=%+v error=%v; expected failed call", result, err) + } + if attempt.Usage.PromptTokens.Known != tc.wantUsage || evidence.Usage.PromptTokens.Known != tc.wantUsage { + t.Fatalf("attempt/aggregate usage = %+v / %+v", attempt.Usage, evidence.Usage) + } + if tc.wantUsage && (attempt.Usage.PromptTokens.Value != 4 || evidence.Usage.PromptTokens.Value != 4) { + t.Fatalf("reported usage was not retained: %+v / %+v", attempt.Usage, evidence.Usage) + } + }) + } +} + +func TestGenerateWithEvidenceRetainsInjectedSuccessAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &evidenceInjectedClient{ + response: &domain.GenerateResponse{Content: "private output"}, + afterGenerate: func() { cancel() }, + } + coordinator := newEvidenceInjectedCoordinator(t, client) + req := testRequest(5) + req.StructuredOutput = &domain.StructuredOutputSpec{} + + result, err, evidence := coordinator.GenerateWithEvidence(ctx, req) + if !errors.Is(err, context.Canceled) || result == nil || result.Content != "private output" { + t.Fatalf("result=%+v error=%v", result, err) + } + if client.calls != 1 || len(evidence.Attempts) != 1 { + t.Fatalf("calls=%d attempts=%+v", client.calls, evidence.Attempts) + } + attempt := evidence.Attempts[0] + if !attempt.ResponseReceived || !attempt.GenerationCompleted || attempt.Failure != domain.ExecutionFailureNone { + t.Fatalf("successful generation evidence was lost: %+v", attempt) + } + if attempt.HTTP != (llm.HTTPExecutionEvidence{}) { + t.Fatalf("injected call claimed HTTP evidence: %+v", attempt.HTTP) + } +} + +type evidenceInjectedClient struct { + calls int + response *domain.GenerateResponse + err error + afterGenerate func() +} + +func (c *evidenceInjectedClient) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { + c.calls++ + if c.afterGenerate != nil { + c.afterGenerate() + } + return c.response, c.err +} + +func newEvidenceInjectedCoordinator(t *testing.T, client llm.Client) *Coordinator { + t.Helper() + manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{ + "limited": {ConcurrencyLimit: 1}, + }) + if err != nil { + t.Fatal(err) + } + return NewInjected(manager, client) +} + type cancelOnDoneContext struct { context.Context cancel context.CancelFunc diff --git a/llm_adapter.go b/llm_adapter.go index 454fca3..219ecf1 100644 --- a/llm_adapter.go +++ b/llm_adapter.go @@ -18,16 +18,17 @@ func (a publicLLMClientAdapter) Generate(ctx context.Context, req domain.Generat if err == nil && resp == nil { err = fmt.Errorf("%w: %w: llm client returned nil response", ErrLLMGenerate, ErrMalformedResponse) } + response := toDomainGenerateResponse(resp) if err != nil { switch { case errors.Is(err, ErrTransport): - return nil, classifiedClientError{cause: err, category: llm.ErrTransport} + return response, classifiedClientError{cause: err, category: llm.ErrTransport} case errors.Is(err, ErrMalformedResponse): - return nil, classifiedClientError{cause: err, category: llm.ErrMalformedResponse} + return response, classifiedClientError{cause: err, category: llm.ErrMalformedResponse} } - return nil, err + return response, err } - return toDomainGenerateResponse(resp), nil + return response, nil } // classifiedClientError makes an explicit public category available internally diff --git a/llm_adapter_internal_test.go b/llm_adapter_internal_test.go index 0cf29a2..9fce767 100644 --- a/llm_adapter_internal_test.go +++ b/llm_adapter_internal_test.go @@ -2,6 +2,7 @@ package promptkit import ( "context" + "errors" "reflect" "strconv" "sync" @@ -65,12 +66,42 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) { } } +func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) { + presence := &TokenUsagePresence{PromptTokens: true} + source := &GenerateResponse{ + Content: "returned with error", + Usage: TokenUsage{PromptTokens: 6}, + UsagePresence: presence, + } + marker := errors.New("injected failure") + adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}} + + got, err := adapter.Generate(context.Background(), domain.GenerateRequest{}) + if got == nil || !errors.Is(err, marker) { + t.Fatalf("response=%+v error=%v", got, err) + } + source.Content = "mutated after return" + presence.PromptTokens = false + if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens { + t.Fatalf("adapter response was not owned: %+v", got) + } +} + type usagePresenceClient struct{ response *GenerateResponse } func (c *usagePresenceClient) Generate(context.Context, GenerateRequest) (*GenerateResponse, error) { return c.response, nil } +type responseAndErrorClient struct { + response *GenerateResponse + err error +} + +func (c responseAndErrorClient) Generate(context.Context, GenerateRequest) (*GenerateResponse, error) { + return c.response, c.err +} + type retainingMutatingLLMClient struct { retained GenerateRequest }