Report actual inference smoke test destinations
This commit is contained in:
@@ -44,8 +44,10 @@ clients.
|
||||
The diagnostic path uses the same private attempt and retry implementation as
|
||||
ordinary generation. It retains invocation-local transport, response,
|
||||
generation, usage, capacity-wait, and retry-delay evidence before normal
|
||||
result-discard and cancellation checks. This evidence is returned to diagnostic
|
||||
orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||
result-discard and cancellation checks. Built-in preparation also supplies its
|
||||
frozen request URL to the evidence path before attempts begin; injected
|
||||
generation has no equivalent HTTP destination. This evidence is returned to
|
||||
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||
observer accounting. The [model-client document](llm.md) describes the
|
||||
transport-observation boundary, and the root
|
||||
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Smoke Test Implementation Review Findings
|
||||
|
||||
Status: one finding completed; two open. Review completed 2026-10-13.
|
||||
Status: two findings completed; one open. Review completed 2026-10-13.
|
||||
|
||||
Reviewed implementation commits after `8fbd058` through
|
||||
`2f735aedf9b5dcc98164292db46ae1eab3e15cde`, against the
|
||||
@@ -56,7 +56,7 @@ fix; `GOWORK=off go test -race ./internal/usecase .`, focused use-case vet, and
|
||||
|
||||
## Finding 2: Report The Actual Initial Inference Request Destination
|
||||
|
||||
Priority: P2. Status: open.
|
||||
Priority: P2. Status: completed 2026-10-13.
|
||||
|
||||
Locations: [`recordGenerationEvidence`](../../internal/usecase/smoke_tester.go),
|
||||
lines 240–248, and
|
||||
@@ -91,6 +91,14 @@ that distinguishes the actual completion URL from the last destination. Reuse
|
||||
existing HTTP redirect tests for credential/response attribution mechanics and
|
||||
keep injected-client HTTP destination evidence unknown.
|
||||
|
||||
Completion: built-in generation now passes the frozen prepared request URL
|
||||
through internal generation evidence. Diagnostic target/initial-attempt URLs
|
||||
use that value, while the configured base stays in effective settings and
|
||||
injected inference leaves HTTP destinations unknown. Public GoDoc and the
|
||||
internal generation document describe the corrected behavior. No-redirect,
|
||||
redirect, and injected regressions failed before the fix; race-enabled tests
|
||||
passed for the facade, model client, generation, use case, and engine packages.
|
||||
|
||||
## Finding 3: Initialize Zero-Dispatch Usage Consistently
|
||||
|
||||
Priority: P3. Status: open.
|
||||
|
||||
@@ -38,6 +38,10 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
|
||||
report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
|
||||
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
|
||||
}
|
||||
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
|
||||
report.Attempts[0].InitialEndpoint != "" || report.Attempts[0].Endpoint != "" {
|
||||
t.Fatalf("injected report claimed HTTP destinations: %+v", report)
|
||||
}
|
||||
|
||||
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",
|
||||
|
||||
@@ -99,6 +99,9 @@ func (c *Coordinator) generate(
|
||||
}
|
||||
|
||||
evidence := GenerationEvidence{}
|
||||
if captureEvidence {
|
||||
evidence.InitialEndpoint = prepared.InitialEndpoint()
|
||||
}
|
||||
var usage observation.UsageAccumulator
|
||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||
if captureEvidence {
|
||||
|
||||
@@ -10,9 +10,12 @@ import (
|
||||
// GenerationEvidence contains invocation-local request evidence without raw
|
||||
// provider bodies, headers, credentials, or generated content.
|
||||
type GenerationEvidence struct {
|
||||
Attempts []AttemptEvidence
|
||||
RetryDelays []time.Duration
|
||||
Usage domain.ObservedTokenUsage
|
||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||
// Injected generation leaves it unknown.
|
||||
InitialEndpoint string
|
||||
Attempts []AttemptEvidence
|
||||
RetryDelays []time.Duration
|
||||
Usage domain.ObservedTokenUsage
|
||||
}
|
||||
|
||||
// AttemptEvidence describes one capacity acquisition and any dispatch it
|
||||
|
||||
@@ -193,6 +193,9 @@ func resolveAPIKey(target domain.ExecutionTarget) string {
|
||||
return apiKey
|
||||
}
|
||||
|
||||
// InitialEndpoint returns the frozen request URL before redirects.
|
||||
func (p *PreparedGeneration) InitialEndpoint() string { return p.endpoint }
|
||||
|
||||
// Execute dispatches one attempt. The returned response is non-nil only when
|
||||
// the successful envelope contains a usable first choice.
|
||||
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
|
||||
|
||||
@@ -73,6 +73,9 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
|
||||
return nil, err
|
||||
}
|
||||
if isInferenceSmokeTest(request.Kind) {
|
||||
// The configured base is not evidence of an HTTP request destination.
|
||||
// Built-in generation supplies its composed URL after preparation.
|
||||
report.Target.Endpoint = ""
|
||||
report.Supported = true
|
||||
report.Applicable = boolPointer(true)
|
||||
} else if err := t.setSupport(report, selection); err != nil {
|
||||
@@ -238,13 +241,14 @@ func (t *SmokeTester) runInference(
|
||||
}
|
||||
|
||||
func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
|
||||
report.Target.Endpoint = evidence.InitialEndpoint
|
||||
report.Usage = evidence.Usage
|
||||
var capacityWait, requestDuration time.Duration
|
||||
for _, attemptEvidence := range evidence.Attempts {
|
||||
capacityWait += attemptEvidence.CapacityWait
|
||||
attempt := domain.SmokeTestAttempt{
|
||||
Dispatched: attemptEvidence.Dispatched,
|
||||
InitialEndpoint: report.Target.Endpoint,
|
||||
InitialEndpoint: evidence.InitialEndpoint,
|
||||
Endpoint: attemptEvidence.HTTP.Endpoint,
|
||||
Facts: generationAttemptFacts(attemptEvidence),
|
||||
Usage: attemptEvidence.Usage,
|
||||
|
||||
@@ -128,7 +128,10 @@ type SmokeTestTarget struct {
|
||||
// prompt, or backend).
|
||||
SelectionSource string
|
||||
// Endpoint is the initially resolved diagnostic destination. It excludes URL
|
||||
// user information; other endpoint components and effective settings may be
|
||||
// user information. For built-in inference it is the prepared completion URL,
|
||||
// not the configured base. It is empty when the HTTP destination is unknown,
|
||||
// including injected inference. The configured base remains available in
|
||||
// EffectiveSettings. Other endpoint components and effective settings may be
|
||||
// sensitive operator data.
|
||||
Endpoint string
|
||||
// EffectiveSettings is the selected target's resolved runtime configuration.
|
||||
@@ -185,7 +188,8 @@ type SmokeTestAttempt struct {
|
||||
// Dispatched distinguishes an actual attempt from an interrupted capacity
|
||||
// acquisition.
|
||||
Dispatched bool
|
||||
// InitialEndpoint is the destination before redirects.
|
||||
// InitialEndpoint is the resolved request destination before redirects;
|
||||
// it is empty for injected inference whose HTTP destination is unknown.
|
||||
InitialEndpoint string
|
||||
// Endpoint is the last observed request destination for this attempt.
|
||||
Endpoint string
|
||||
|
||||
@@ -419,8 +419,52 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
|
||||
report.Target.ProfileID != "profile-b" || report.Target.SelectionSource != "backend_default" {
|
||||
t.Fatalf("unexpected cross-backend target: %+v", report.Target)
|
||||
}
|
||||
if report.Target.Endpoint != endpointOverride || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
|
||||
observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 || report.Attempts[0].Endpoint != observedURL {
|
||||
if report.Target.Endpoint != observedURL || report.Target.EffectiveSettings.Endpoint != endpointOverride ||
|
||||
observedURL != endpointOverride+"/chat/completions" || len(report.Attempts) != 1 ||
|
||||
report.Attempts[0].InitialEndpoint != observedURL || report.Attempts[0].Endpoint != observedURL {
|
||||
t.Fatalf("endpoint override not used by facade: target=%+v attempt=%+v URL=%q", report.Target, report.Attempts, observedURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t *testing.T) {
|
||||
const base = "http://initial.example/v1"
|
||||
const redirected = "http://redirect.example/completion"
|
||||
var observed []string
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
observed = append(observed, request.URL.String())
|
||||
if len(observed) == 1 {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusTemporaryRedirect,
|
||||
Header: http.Header{"Location": []string{redirected}},
|
||||
Body: io.NopCloser(strings.NewReader("")),
|
||||
Request: request,
|
||||
}, nil
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"unexpected"}}]}`)),
|
||||
Request: request,
|
||||
}, nil
|
||||
})}
|
||||
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
|
||||
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
|
||||
Kind: promptkit.SmokeTestTextInference, ProfileID: "profile",
|
||||
})
|
||||
if err != nil || report == nil || !report.Completed || len(report.Attempts) != 1 || len(observed) != 2 {
|
||||
t.Fatalf("report=%+v error=%v observed=%v", report, err, observed)
|
||||
}
|
||||
if observed[0] != base+"/chat/completions" || observed[1] != redirected ||
|
||||
report.Target.Endpoint != observed[0] || report.Target.EffectiveSettings.Endpoint != base ||
|
||||
report.Attempts[0].InitialEndpoint != observed[0] || report.Attempts[0].Endpoint != observed[1] {
|
||||
t.Fatalf("incorrect redirect attribution: target=%+v attempt=%+v observed=%v", report.Target, report.Attempts[0], observed)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user