Report actual inference smoke test destinations

This commit is contained in:
2026-10-13 20:27:46 +00:00
parent 208ff42915
commit 116bd0e39b
9 changed files with 87 additions and 12 deletions

View File

@@ -44,8 +44,10 @@ clients.
The diagnostic path uses the same private attempt and retry implementation as The diagnostic path uses the same private attempt and retry implementation as
ordinary generation. It retains invocation-local transport, response, ordinary generation. It retains invocation-local transport, response,
generation, usage, capacity-wait, and retry-delay evidence before normal generation, usage, capacity-wait, and retry-delay evidence before normal
result-discard and cancellation checks. This evidence is returned to diagnostic result-discard and cancellation checks. Built-in preparation also supplies its
orchestration; it is not added to public `ExecutionEvent` values or ordinary frozen request URL to the evidence path before attempts begin; injected
generation has no equivalent HTTP destination. This evidence is returned to
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
observer accounting. The [model-client document](llm.md) describes the observer accounting. The [model-client document](llm.md) describes the
transport-observation boundary, and the root transport-observation boundary, and the root
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics. [`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.

View File

@@ -1,6 +1,6 @@
# Smoke Test Implementation Review Findings # Smoke Test Implementation Review Findings
Status: one finding completed; two open. Review completed 2026-10-13. Status: two findings completed; one open. Review completed 2026-10-13.
Reviewed implementation commits after `8fbd058` through Reviewed implementation commits after `8fbd058` through
`2f735aedf9b5dcc98164292db46ae1eab3e15cde`, against the `2f735aedf9b5dcc98164292db46ae1eab3e15cde`, against the
@@ -56,7 +56,7 @@ fix; `GOWORK=off go test -race ./internal/usecase .`, focused use-case vet, and
## Finding 2: Report The Actual Initial Inference Request Destination ## Finding 2: Report The Actual Initial Inference Request Destination
Priority: P2. Status: open. Priority: P2. Status: completed 2026-10-13.
Locations: [`recordGenerationEvidence`](../../internal/usecase/smoke_tester.go), Locations: [`recordGenerationEvidence`](../../internal/usecase/smoke_tester.go),
lines 240–248, and lines 240–248, and
@@ -91,6 +91,14 @@ that distinguishes the actual completion URL from the last destination. Reuse
existing HTTP redirect tests for credential/response attribution mechanics and existing HTTP redirect tests for credential/response attribution mechanics and
keep injected-client HTTP destination evidence unknown. keep injected-client HTTP destination evidence unknown.
Completion: built-in generation now passes the frozen prepared request URL
through internal generation evidence. Diagnostic target/initial-attempt URLs
use that value, while the configured base stays in effective settings and
injected inference leaves HTTP destinations unknown. Public GoDoc and the
internal generation document describe the corrected behavior. No-redirect,
redirect, and injected regressions failed before the fix; race-enabled tests
passed for the facade, model client, generation, use case, and engine packages.
## Finding 3: Initialize Zero-Dispatch Usage Consistently ## Finding 3: Initialize Zero-Dispatch Usage Consistently
Priority: P3. Status: open. Priority: P3. Status: open.

View File

@@ -38,6 +38,10 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" { report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
t.Fatalf("injected smoke report calls=%d: %+v", calls, report) t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
} }
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
}
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome", Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",

View File

@@ -99,6 +99,9 @@ func (c *Coordinator) generate(
} }
evidence := GenerationEvidence{} evidence := GenerationEvidence{}
if captureEvidence {
evidence.InitialEndpoint = prepared.InitialEndpoint()
}
var usage observation.UsageAccumulator var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) { finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence { if captureEvidence {

View File

@@ -10,6 +10,9 @@ import (
// GenerationEvidence contains invocation-local request evidence without raw // GenerationEvidence contains invocation-local request evidence without raw
// provider bodies, headers, credentials, or generated content. // provider bodies, headers, credentials, or generated content.
type GenerationEvidence struct { type GenerationEvidence struct {
// InitialEndpoint is the frozen built-in request URL, if prepared.
// Injected generation leaves it unknown.
InitialEndpoint string
Attempts []AttemptEvidence Attempts []AttemptEvidence
RetryDelays []time.Duration RetryDelays []time.Duration
Usage domain.ObservedTokenUsage Usage domain.ObservedTokenUsage

View File

@@ -193,6 +193,9 @@ func resolveAPIKey(target domain.ExecutionTarget) string {
return apiKey return apiKey
} }
// InitialEndpoint returns the frozen request URL before redirects.
func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint }
// Execute dispatches one attempt. The returned response is non-nil only when // Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice. // the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome { func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {

View File

@@ -73,6 +73,9 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
return nil, err return nil, err
} }
if isInferenceSmokeTest(request.Kind) { if isInferenceSmokeTest(request.Kind) {
// The configured base is not evidence of an HTTP request destination.
// Built-in generation supplies its composed URL after preparation.
report.Target.Endpoint = ""
report.Supported = true report.Supported = true
report.Applicable = boolPointer(true) report.Applicable = boolPointer(true)
} else if err := t.setSupport(report, selection); err != nil { } else if err := t.setSupport(report, selection); err != nil {
@@ -238,13 +241,14 @@ func (t *SmokeTester) runInference(
} }
func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) { func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
report.Target.Endpoint = evidence.InitialEndpoint
report.Usage = evidence.Usage report.Usage = evidence.Usage
var capacityWait, requestDuration time.Duration var capacityWait, requestDuration time.Duration
for _, attemptEvidence := range evidence.Attempts { for _, attemptEvidence := range evidence.Attempts {
capacityWait += attemptEvidence.CapacityWait capacityWait += attemptEvidence.CapacityWait
attempt := domain.SmokeTestAttempt{ attempt := domain.SmokeTestAttempt{
Dispatched: attemptEvidence.Dispatched, Dispatched: attemptEvidence.Dispatched,
InitialEndpoint: report.Target.Endpoint, InitialEndpoint: evidence.InitialEndpoint,
Endpoint: attemptEvidence.HTTP.Endpoint, Endpoint: attemptEvidence.HTTP.Endpoint,
Facts: generationAttemptFacts(attemptEvidence), Facts: generationAttemptFacts(attemptEvidence),
Usage: attemptEvidence.Usage, Usage: attemptEvidence.Usage,

View File

@@ -128,7 +128,10 @@ type SmokeTestTarget struct {
// prompt, or backend). // prompt, or backend).
SelectionSource string SelectionSource string
// Endpoint is the initially resolved diagnostic destination. It excludes URL // Endpoint is the initially resolved diagnostic destination. It excludes URL
// user information; other endpoint components and effective settings may be // user information. For built-in inference it is the prepared completion URL,
// not the configured base. It is empty when the HTTP destination is unknown,
// including injected inference. The configured base remains available in
// EffectiveSettings. Other endpoint components and effective settings may be
// sensitive operator data. // sensitive operator data.
Endpoint string Endpoint string
// EffectiveSettings is the selected target's resolved runtime configuration. // EffectiveSettings is the selected target's resolved runtime configuration.
@@ -185,7 +188,8 @@ type SmokeTestAttempt struct {
// Dispatched distinguishes an actual attempt from an interrupted capacity // Dispatched distinguishes an actual attempt from an interrupted capacity
// acquisition. // acquisition.
Dispatched bool Dispatched bool
// InitialEndpoint is the destination before redirects. // InitialEndpoint is the resolved request destination before redirects;
// it is empty for injected inference whose HTTP destination is unknown.
InitialEndpoint string InitialEndpoint string
// Endpoint is the last observed request destination for this attempt. // Endpoint is the last observed request destination for this attempt.
Endpoint string Endpoint string

View File

@@ -419,8 +419,52 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" { report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" {
t.Fatalf("unexpected cross-backend target: %+v", report.Target) t.Fatalf("unexpected cross-backend target: %+v", report.Target)
} }
if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride || if report.Target.Endpoint != observedURL || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL { observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 ||
report.Attempts[0].InitialEndpoint != observedURL || report.Attempts[0].Endpoint != observedURL {
t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL) t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL)
} }
} }
func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t *testing.T) {
const base = "http://initial.example/v1"
const redirected = "http://redirect.example/completion"
var observed []string
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
observed = append(observed, request.URL.String())
if len(observed) == 1 {
return &http.Response{
StatusCode: http.StatusTemporaryRedirect,
Header: http.Header{"Location": []string{redirected}},
Body: io.NopCloser(strings.NewReader("")),
Request: request,
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"unexpected"}}]}`)),
Request: request,
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,
})
if err != nil {
t.Fatal(err)
}
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
})
if err != nil || report == nil || !report.Completed || len(report.Attempts) != 1 || len(observed) != 2 {
t.Fatalf("report=%+v error=%v observed=%v", report, err, observed)
}
if observed[0] != base+"/chat/completions" || observed[1] != redirected ||
report.Target.Endpoint != observed[0] || report.Target.EffectiveSettings.Endpoint != base ||
report.Attempts[0].InitialEndpoint != observed[0] || report.Attempts[0].Endpoint != observed[1] {
t.Fatalf("incorrect redirect attribution: target=%+v attempt=%+v observed=%v", report.Target, report.Attempts[0], observed)
}
}