Report actual inference smoke test destinations

This commit is contained in:
2026-10-13 20:27:46 +00:00
parent 208ff42915
commit 116bd0e39b
9 changed files with 87 additions and 12 deletions

View File

@@ -44,8 +44,10 @@ clients.
The diagnostic path uses the same private attempt and retry implementation as
ordinary generation. It retains invocation-local transport, response,
generation, usage, capacity-wait, and retry-delay evidence before normal
result-discard and cancellation checks. This evidence is returned to diagnostic
orchestration; it is not added to public `ExecutionEvent` values or ordinary
result-discard and cancellation checks. Built-in preparation also supplies its
frozen request URL to the evidence path before attempts begin; injected
generation has no equivalent HTTP destination. This evidence is returned to
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
observer accounting. The [model-client document](llm.md) describes the
transport-observation boundary, and the root
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.

View File

@@ -1,6 +1,6 @@
# Smoke Test Implementation Review Findings
Status: one finding completed; two open. Review completed 2026-10-13.
Status: two findings completed; one open. Review completed 2026-10-13.
Reviewed implementation commits after `8fbd058` through
`2f735aedf9b5dcc98164292db46ae1eab3e15cde`, against the
@@ -56,7 +56,7 @@ fix; `GOWORK=off go test -race ./internal/usecase .`, focused use-case vet, and
## Finding 2: Report The Actual Initial Inference Request Destination
Priority: P2. Status: open.
Priority: P2. Status: completed 2026-10-13.
Locations: [`recordGenerationEvidence`](../../internal/usecase/smoke_tester.go),
lines 240–248, and
@@ -91,6 +91,14 @@ that distinguishes the actual completion URL from the last destination. Reuse
existing HTTP redirect tests for credential/response attribution mechanics and
keep injected-client HTTP destination evidence unknown.
Completion: built-in generation now passes the frozen prepared request URL
through internal generation evidence. Diagnostic target/initial-attempt URLs
use that value, while the configured base stays in effective settings and
injected inference leaves HTTP destinations unknown. Public GoDoc and the
internal generation document describe the corrected behavior. No-redirect,
redirect, and injected regressions failed before the fix; race-enabled tests
passed for the facade, model client, generation, use case, and engine packages.
## Finding 3: Initialize Zero-Dispatch Usage Consistently
Priority: P3. Status: open.

View File

@@ -38,6 +38,10 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
}
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
}
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",

View File

@@ -99,6 +99,9 @@ func (c *Coordinator) generate(
}
evidence := GenerationEvidence{}
if captureEvidence {
evidence.InitialEndpoint = prepared.InitialEndpoint()
}
var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence {

View File

@@ -10,9 +10,12 @@ import (
// GenerationEvidence contains invocation-local request evidence without raw
// provider bodies, headers, credentials, or generated content.
type GenerationEvidence struct {
Attempts []AttemptEvidence
RetryDelays []time.Duration
Usage domain.ObservedTokenUsage
// InitialEndpoint is the frozen built-in request URL, if prepared.
// Injected generation leaves it unknown.
InitialEndpoint string
Attempts []AttemptEvidence
RetryDelays []time.Duration
Usage domain.ObservedTokenUsage
}
// AttemptEvidence describes one capacity acquisition and any dispatch it

View File

@@ -193,6 +193,9 @@ func resolveAPIKey(target domain.ExecutionTarget) string {
return apiKey
}
// InitialEndpoint returns the frozen request URL before redirects.
func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint }
// Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {

View File

@@ -73,6 +73,9 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
return nil, err
}
if isInferenceSmokeTest(request.Kind) {
// The configured base is not evidence of an HTTP request destination.
// Built-in generation supplies its composed URL after preparation.
report.Target.Endpoint = ""
report.Supported = true
report.Applicable = boolPointer(true)
} else if err := t.setSupport(report, selection); err != nil {
@@ -238,13 +241,14 @@ func (t *SmokeTester) runInference(
}
func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
report.Target.Endpoint = evidence.InitialEndpoint
report.Usage = evidence.Usage
var capacityWait, requestDuration time.Duration
for _, attemptEvidence := range evidence.Attempts {
capacityWait += attemptEvidence.CapacityWait
attempt := domain.SmokeTestAttempt{
Dispatched: attemptEvidence.Dispatched,
InitialEndpoint: report.Target.Endpoint,
InitialEndpoint: evidence.InitialEndpoint,
Endpoint: attemptEvidence.HTTP.Endpoint,
Facts: generationAttemptFacts(attemptEvidence),
Usage: attemptEvidence.Usage,

View File

@@ -128,7 +128,10 @@ type SmokeTestTarget struct {
// prompt, or backend).
SelectionSource string
// Endpoint is the initially resolved diagnostic destination. It excludes URL
// user information; other endpoint components and effective settings may be
// user information. For built-in inference it is the prepared completion URL,
// not the configured base. It is empty when the HTTP destination is unknown,
// including injected inference. The configured base remains available in
// EffectiveSettings. Other endpoint components and effective settings may be
// sensitive operator data.
Endpoint string
// EffectiveSettings is the selected target's resolved runtime configuration.
@@ -185,7 +188,8 @@ type SmokeTestAttempt struct {
// Dispatched distinguishes an actual attempt from an interrupted capacity
// acquisition.
Dispatched bool
// InitialEndpoint is the destination before redirects.
// InitialEndpoint is the resolved request destination before redirects;
// it is empty for injected inference whose HTTP destination is unknown.
InitialEndpoint string
// Endpoint is the last observed request destination for this attempt.
Endpoint string

View File

@@ -419,8 +419,52 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" {
t.Fatalf("unexpected cross-backend target: %+v", report.Target)
}
if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL {
if report.Target.Endpoint != observedURL || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 ||
report.Attempts[0].InitialEndpoint != observedURL || report.Attempts[0].Endpoint != observedURL {
t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL)
}
}
func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t *testing.T) {
const base = "http://initial.example/v1"
const redirected = "http://redirect.example/completion"
var observed []string
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
observed = append(observed, request.URL.String())
if len(observed) == 1 {
return &http.Response{
StatusCode: http.StatusTemporaryRedirect,
Header: http.Header{"Location": []string{redirected}},
Body: io.NopCloser(strings.NewReader("")),
Request: request,
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"unexpected"}}]}`)),
Request: request,
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,
})
if err != nil {
t.Fatal(err)
}
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
})
if err != nil || report == nil || !report.Completed || len(report.Attempts) != 1 || len(observed) != 2 {
t.Fatalf("report=%+v error=%v observed=%v", report, err, observed)
}
if observed[0] != base+"/chat/completions" || observed[1] != redirected ||
report.Target.Endpoint != observed[0] || report.Target.EffectiveSettings.Endpoint != base ||
report.Attempts[0].InitialEndpoint != observed[0] || report.Attempts[0].Endpoint != observed[1] {
t.Fatalf("incorrect redirect attribution: target=%+v attempt=%+v observed=%v", report.Target, report.Attempts[0], observed)
}
}