From 116bd0e39be64c0c068e73b04f406435495b6d15 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 13 Oct 2026 20:27:46 +0000 Subject: [PATCH] Report actual inference smoke test destinations --- docs/internal/generation.md | 6 ++- docs/roadmap/remediation.md | 12 +++++- internal/engine/smoke_test.go | 4 ++ internal/generation/coordinator.go | 3 ++ internal/generation/evidence.go | 9 +++-- internal/llm/openai_compatible_client.go | 3 ++ internal/usecase/smoke_tester.go | 6 ++- smoke_diagnostics.go | 8 +++- smoke_integration_contract_test.go | 48 +++++++++++++++++++++++- 9 files changed, 87 insertions(+), 12 deletions(-) diff --git a/docs/internal/generation.md b/docs/internal/generation.md index 887a5c7..4437c15 100644 --- a/docs/internal/generation.md +++ b/docs/internal/generation.md @@ -44,8 +44,10 @@ clients. The diagnostic path uses the same private attempt and retry implementation as ordinary generation. It retains invocation-local transport, response, generation, usage, capacity-wait, and retry-delay evidence before normal -result-discard and cancellation checks. This evidence is returned to diagnostic -orchestration; it is not added to public `ExecutionEvent` values or ordinary +result-discard and cancellation checks. Built-in preparation also supplies its +frozen request URL to the evidence path before attempts begin; injected +generation has no equivalent HTTP destination. This evidence is returned to +diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary observer accounting. The [model-client document](llm.md) describes the transport-observation boundary, and the root [`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics. diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index eeb8b4d..03dc81c 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -1,6 +1,6 @@ # Smoke Test Implementation Review Findings -Status: one finding completed; two open. Review completed 2026-10-13. +Status: two findings completed; one open. Review completed 2026-10-13. Reviewed implementation commits after `8fbd058` through `2f735aedf9b5dcc98164292db46ae1eab3e15cde`, against the @@ -56,7 +56,7 @@ fix; `GOWORK=off go test -race ./internal/usecase .`, focused use-case vet, and ## Finding 2: Report The Actual Initial Inference Request Destination -Priority: P2. Status: open. +Priority: P2. Status: completed 2026-10-13. Locations: [`recordGenerationEvidence`](../../internal/usecase/smoke_tester.go), lines 240–248, and @@ -91,6 +91,14 @@ that distinguishes the actual completion URL from the last destination. Reuse existing HTTP redirect tests for credential/response attribution mechanics and keep injected-client HTTP destination evidence unknown. +Completion: built-in generation now passes the frozen prepared request URL +through internal generation evidence. Diagnostic target/initial-attempt URLs +use that value, while the configured base stays in effective settings and +injected inference leaves HTTP destinations unknown. Public GoDoc and the +internal generation document describe the corrected behavior. No-redirect, +redirect, and injected regressions failed before the fix; race-enabled tests +passed for the facade, model client, generation, use case, and engine packages. + ## Finding 3: Initialize Zero-Dispatch Usage Consistently Priority: P3. Status: open. diff --git a/internal/engine/smoke_test.go b/internal/engine/smoke_test.go index b4e4bfe..5674256 100644 --- a/internal/engine/smoke_test.go +++ b/internal/engine/smoke_test.go @@ -38,6 +38,10 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" { t.Fatalf("injected smoke report calls=%d: %+v", calls, report) } + if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 || + report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" { + t.Fatalf("injected report claimed HTTP destinations: %+v", report) + } authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome", diff --git a/internal/generation/coordinator.go b/internal/generation/coordinator.go index aff2f12..58d870d 100644 --- a/internal/generation/coordinator.go +++ b/internal/generation/coordinator.go @@ -99,6 +99,9 @@ func (c *Coordinator) generate( } evidence := GenerationEvidence{} + if captureEvidence { + evidence.InitialEndpoint = prepared.InitialEndpoint() + } var usage observation.UsageAccumulator finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) { if captureEvidence { diff --git a/internal/generation/evidence.go b/internal/generation/evidence.go index 60ab891..eec2878 100644 --- a/internal/generation/evidence.go +++ b/internal/generation/evidence.go @@ -10,9 +10,12 @@ import ( // GenerationEvidence contains invocation-local request evidence without raw // provider bodies, headers, credentials, or generated content. type GenerationEvidence struct { - Attempts []AttemptEvidence - RetryDelays []time.Duration - Usage domain.ObservedTokenUsage + // InitialEndpoint is the frozen built-in request URL, if prepared. + // Injected generation leaves it unknown. + InitialEndpoint string + Attempts []AttemptEvidence + RetryDelays []time.Duration + Usage domain.ObservedTokenUsage } // AttemptEvidence describes one capacity acquisition and any dispatch it diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index f82f8dc..a469d97 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -193,6 +193,9 @@ func resolveAPIKey(target domain.ExecutionTarget) string { return apiKey } +// InitialEndpoint returns the frozen request URL before redirects. +func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint } + // Execute dispatches one attempt. The returned response is non-nil only when // the successful envelope contains a usable first choice. func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { diff --git a/internal/usecase/smoke_tester.go b/internal/usecase/smoke_tester.go index a22c93c..95ac751 100644 --- a/internal/usecase/smoke_tester.go +++ b/internal/usecase/smoke_tester.go @@ -73,6 +73,9 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) return nil, err } if isInferenceSmokeTest(request.Kind) { + // The configured base is not evidence of an HTTP request destination. + // Built-in generation supplies its composed URL after preparation. + report.Target.Endpoint = "" report.Supported = true report.Applicable = boolPointer(true) } else if err := t.setSupport(report, selection); err != nil { @@ -238,13 +241,14 @@ func (t *SmokeTester) runInference( } func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) { + report.Target.Endpoint = evidence.InitialEndpoint report.Usage = evidence.Usage var capacityWait, requestDuration time.Duration for _, attemptEvidence := range evidence.Attempts { capacityWait += attemptEvidence.CapacityWait attempt := domain.SmokeTestAttempt{ Dispatched: attemptEvidence.Dispatched, - InitialEndpoint: report.Target.Endpoint, + InitialEndpoint: evidence.InitialEndpoint, Endpoint: attemptEvidence.HTTP.Endpoint, Facts: generationAttemptFacts(attemptEvidence), Usage: attemptEvidence.Usage, diff --git a/smoke_diagnostics.go b/smoke_diagnostics.go index d447d0d..2edbb48 100644 --- a/smoke_diagnostics.go +++ b/smoke_diagnostics.go @@ -128,7 +128,10 @@ type SmokeTestTarget struct { // prompt, or backend). SelectionSource string // Endpoint is the initially resolved diagnostic destination. It excludes URL - // user information; other endpoint components and effective settings may be + // user information. For built-in inference it is the prepared completion URL, + // not the configured base. It is empty when the HTTP destination is unknown, + // including injected inference. The configured base remains available in + // EffectiveSettings. Other endpoint components and effective settings may be // sensitive operator data. Endpoint string // EffectiveSettings is the selected target's resolved runtime configuration. @@ -185,7 +188,8 @@ type SmokeTestAttempt struct { // Dispatched distinguishes an actual attempt from an interrupted capacity // acquisition. Dispatched bool - // InitialEndpoint is the destination before redirects. + // InitialEndpoint is the resolved request destination before redirects; + // it is empty for injected inference whose HTTP destination is unknown. InitialEndpoint string // Endpoint is the last observed request destination for this attempt. Endpoint string diff --git a/smoke_integration_contract_test.go b/smoke_integration_contract_test.go index 19d3db3..8b1a6e8 100644 --- a/smoke_integration_contract_test.go +++ b/smoke_integration_contract_test.go @@ -419,8 +419,52 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" { t.Fatalf("unexpected cross-backend target: %+v", report.Target) } - if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride || - observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL { + if report.Target.Endpoint != observedURL || report.Target.EffectiveSettings.Endpoint != endpointOverride || + observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || + report.Attempts[0].InitialEndpoint != observedURL || report.Attempts[0].Endpoint != observedURL { t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL) } } + +func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t *testing.T) { + const base = "http://initial.example/v1" + const redirected = "http://redirect.example/completion" + var observed []string + client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + observed = append(observed, request.URL.String()) + if len(observed) == 1 { + return &http.Response{ + StatusCode: http.StatusTemporaryRedirect, + Header: http.Header{"Location": []string{redirected}}, + Body: io.NopCloser(strings.NewReader("")), + Request: request, + }, nil + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"unexpected"}}]}`)), + Request: request, + }, nil + })} + input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}}, + })} + engine, err := promptkit.NewEngine(promptkit.Config{ + Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client, + }) + if err != nil { + t.Fatal(err) + } + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", + }) + if err != nil || report == nil || !report.Completed || len(report.Attempts) != 1 || len(observed) != 2 { + t.Fatalf("report=%+v error=%v observed=%v", report, err, observed) + } + if observed[0] != base+"/chat/completions" || observed[1] != redirected || + report.Target.Endpoint != observed[0] || report.Target.EffectiveSettings.Endpoint != base || + report.Attempts[0].InitialEndpoint != observed[0] || report.Attempts[0].Endpoint != observed[1] { + t.Fatalf("incorrect redirect attribution: target=%+v attempt=%+v observed=%v", report.Target, report.Attempts[0], observed) + } +}