Integrate inference smoke diagnostics

docs/roadmap/implementation.md

@@ -622,6 +622,14 @@ Validate: focused use-case/engine tests for text/schema results, retry-default
 isolation, injected generation, required-credential/capacity findings, and evidence
 after cancellation. Stage 17 owns broader facade-level capacity/observer regressions.
 
+**Completed: 2026-10-13.** Integrated private inference cases through the shared
+generation coordinator, preserving attempts, usage, phase timings, last-dispatched
+facts, and local validation findings. Required-credential and logical-capacity
+findings conclude without dispatch. Built-in and injected engines share the
+runner/coordinator while only built-in engines supply HTTP diagnostics.
+Validation passed: `go test -race ./internal/usecase ./internal/generation
+./internal/engine`, `go vet ./...`, `go test ./...`, and `git diff --check`.
+
 ## Stage 16: Expose The Public SmokeTest Facade
 
 Scope: root declarations, GoDoc, conversions, and external-package API tests.

internal/engine/engine.go

@@ -81,9 +81,10 @@ func (o *Options) SetSchemaFile(path string) error {
 
 // Engine forwards domain operations to the single assembled use-case runner.
 type Engine struct {
-	runner     *usecase.Runner
+	runner      *usecase.Runner
-	inspection configuration.Inspection
+	smokeTester *usecase.SmokeTester
-	listing    identityInventory
+	inspection  configuration.Inspection
+	listing     identityInventory
 }
 
 type identityInventory interface {
@@ -124,11 +125,13 @@ func New(cfg Config, options Options) (*Engine, error) {
 		validator = validate.NewStandardValidator(schemaDir)
 	}
 	var client llm.Client
+	var diagnosticHTTPClient *llm.OpenAICompatibleClient
 	if options.llmClient == nil {
 		builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
 		if newErr != nil {
 			return nil, newErr
 		}
+		diagnosticHTTPClient = builtIn
 		client = generation.NewBuiltIn(manager, builtIn)
 	} else {
 		client = generation.NewInjected(manager, options.llmClient)
@@ -138,10 +141,12 @@ func New(cfg Config, options Options) (*Engine, error) {
 		reader = artifact.NewCompositeReader()
 	}
 
+	runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)
 	return &Engine{
-		runner:     usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
+		runner:      runner,
-		inspection: snapshot.Inspection(),
+		smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
-		listing:    snapshot,
+		inspection:  snapshot.Inspection(),
+		listing:     snapshot,
 	}, nil
 }
 
@@ -214,3 +219,10 @@ func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.Ru
 func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) {
 	return e.runner.RunPrepared(ctx, prepared)
 }
+
+func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
+	if e == nil || e.smokeTester == nil {
+		return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest)
+	}
+	return e.smokeTester.Run(ctx, request)
+}

internal/engine/smoke_test.go

@@ -0,0 +1,101 @@
+package engine
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
+	calls := 0
+	options := Options{}
+	options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
+		calls++
+		if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" {
+			t.Fatalf("injected generation request = %+v", request)
+		}
+		return &domain.GenerateResponse{Content: "engine response"}, nil
+	}))
+	assembled, err := New(Config{}, options)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable ||
+		report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
+		report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
+		report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
+		report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
+		t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
+	}
+
+	authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable ||
+		authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 {
+		t.Fatalf("injected authentication report = %+v", authentication)
+	}
+}
+
+func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) {
+	var observedURL string
+	var authorization string
+	options := Options{}
+	assembled, err := New(Config{HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
+		observedURL = request.URL.String()
+		authorization = request.Header.Get("Authorization")
+		return &http.Response{
+			StatusCode: http.StatusOK,
+			Header:     make(http.Header),
+			Body:       io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)),
+		}, nil
+	})}}, options)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if observedURL == "" || authorization != "" {
+		t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization)
+	}
+	if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
+		report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
+		report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
+		report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied ||
+		report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
+		report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 ||
+		report.Attempts[0].Endpoint != observedURL {
+		t.Fatalf("built-in smoke report = %+v", report)
+	}
+}
+
+type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
+
+func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	return f(ctx, request)
+}
+
+type engineRoundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return f(request)
+}

internal/generation/coordinator.go

@@ -117,12 +117,13 @@ func (c *Coordinator) generate(
 			return finish(nil, err)
 		}
 		ordinal := attempts + 1
-		outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
+		outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
 		if dispatched {
 			attempts++
 		}
 		if captureEvidence {
 			attempt := AttemptEvidence{
+				StartedAt:           startedAt.UTC(),
 				Dispatched:          dispatched,
 				ResponseReceived:    outcome.Evidence.ResponseReceived,
 				HTTP:                cloneHTTPEvidence(outcome.Evidence),
@@ -220,6 +221,7 @@ func (c *Coordinator) generateInjected(
 	if err != nil {
 		if captureEvidence {
 			evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
+				StartedAt:    start.UTC(),
 				CapacityWait: capacityWait,
 				Failure:      ClassifyFailure(err),
 			})
@@ -234,6 +236,7 @@ func (c *Coordinator) generateInjected(
 		release()
 		if captureEvidence {
 			evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
+				StartedAt:    start.UTC(),
 				CapacityWait: capacityWait,
 				Failure:      ClassifyFailure(err),
 			})
@@ -251,6 +254,7 @@ func (c *Coordinator) generateInjected(
 	}
 	if captureEvidence {
 		attempt := AttemptEvidence{
+			StartedAt:           start.UTC(),
 			Dispatched:          true,
 			ResponseReceived:    response != nil,
 			RequestDuration:     duration,
@@ -293,16 +297,17 @@ func (c *Coordinator) generateInjected(
 	return finish(response, nil)
 }
 
-func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) {
+func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
 	start := c.now()
+	startedAt = start
 	release, err := c.manager.Acquire(ctx, backendID)
 	capacityWait = c.capacityWait(backendID, start)
 	if err != nil {
-		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
+		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
 	}
 	defer release()
 	if err := ctx.Err(); err != nil {
-		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0
+		return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
 	}
 	dispatchStart := c.now()
 	if captureEvidence {
@@ -312,7 +317,7 @@ func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *l
 		outcome = prepared.Execute(ctx)
 		dispatched = true
 	}
-	return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now())
+	return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
 }
 
 func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {

internal/generation/evidence.go

@@ -18,6 +18,7 @@ type GenerationEvidence struct {
 // AttemptEvidence describes one capacity acquisition and any dispatch it
 // reached. Generated content remains on the separate response return value.
 type AttemptEvidence struct {
+	StartedAt                 time.Time
 	Dispatched                bool
 	ResponseReceived          bool
 	HTTP                      llm.HTTPExecutionEvidence

internal/usecase/smoke_inference_test.go

@@ -0,0 +1,288 @@
+package usecase
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/generation"
+	"gitea.maximumdirect.net/eric/promptkit/internal/llm"
+)
+
+func TestSmokeTesterRunsPrivateInferenceCasesThroughInjectedCoordinator(t *testing.T) {
+	for _, kind := range []domain.SmokeTestKind{domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput} {
+		t.Run(string(kind), func(t *testing.T) {
+			runner := smokeTesterRunner(t, smokeSelectionConfig())
+			request := domain.SmokeTestRequest{Kind: kind, ProfileID: "alpha-profile"}
+			selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			privateCase, err := prepareSmokeInferenceCase(context.Background(), kind, selection.target, selection.transientRetries)
+			if err != nil {
+				t.Fatal(err)
+			}
+			calls := 0
+			client := smokeInferenceClientFunc(func(_ context.Context, generationRequest domain.GenerateRequest) (*domain.GenerateResponse, error) {
+				calls++
+				if generationRequest.TransientRetries != 0 || generationRequest.Generation != 1 || generationRequest.Target.BackendID != "alpha" {
+					t.Fatalf("generation request selection/retries = %+v", generationRequest)
+				}
+				return &domain.GenerateResponse{Content: privateCase.expectedContent}, nil
+			})
+			manager, err := capacity.NewManager(nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			runner.llm = generation.NewInjected(manager, client)
+			runner.admitter = &smokeAdmissionSpy{}
+			runner.defaultTransientRetries = 5
+
+			report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if calls != 1 || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone {
+				t.Fatalf("calls=%d report=%+v", calls, report)
+			}
+			if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
+				report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.EndpointReachable != nil {
+				t.Fatalf("generation facts = %+v", report.Facts)
+			}
+			if kind == domain.SmokeTestTextInference {
+				if report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || !*report.Facts.ExpectedTextMatched {
+					t.Fatalf("text facts = %+v", report.Facts)
+				}
+			} else if report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested ||
+				report.Facts.ValidJSON == nil || !*report.Facts.ValidJSON || report.Facts.SchemaMatched == nil || !*report.Facts.SchemaMatched {
+				t.Fatalf("structured facts = %+v", report.Facts)
+			}
+			if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].Timing.StartedAt.IsZero() ||
+				report.Attempts[0].Timing.CapacityWait == nil || report.Attempts[0].Timing.RequestDuration == nil ||
+				report.Timing.CapacityWait == nil || report.Timing.RequestDuration == nil || report.Timing.EvaluationDuration == nil {
+				t.Fatalf("attempt/timing evidence = %+v", report)
+			}
+			if report.Usage.PromptTokens.Known || report.Attempts[0].Usage.PromptTokens.Known {
+				t.Fatalf("unexpected usage evidence = %+v", report.Usage)
+			}
+		})
+	}
+}
+
+func TestSmokeTesterConcludesMissingRequiredCredentialWithoutDispatch(t *testing.T) {
+	values := smokeSelectionConfig()
+	profiles := values["profiles"].(map[string]any)
+	profiles["required-profile"] = map[string]any{
+		"backend": "alpha", "model": "required-model", "api_key_required": true,
+	}
+	runner := smokeTesterRunner(t, values)
+	admitter := &smokeAdmissionSpy{}
+	runner.admitter = admitter
+
+	report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, ProfileID: "required-profile",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed ||
+		report.Reason != domain.SmokeTestReasonRequiredCredentialMissing || report.Failure != domain.ExecutionFailureInvalidRequest {
+		t.Fatalf("missing-credential report = %+v", report)
+	}
+	if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || admitter.calls != 0 ||
+		!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
+		t.Fatalf("missing credential invented dispatch evidence: %+v admission calls=%d", report, admitter.calls)
+	}
+}
+
+func TestSmokeTesterReportsStructuredContentValidationFindings(t *testing.T) {
+	runner := smokeTesterRunner(t, smokeSelectionConfig())
+	request := domain.SmokeTestRequest{Kind: domain.SmokeTestStructuredOutput, ProfileID: "alpha-profile"}
+	selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	privateCase, err := prepareSmokeInferenceCase(context.Background(), request.Kind, selection.target, selection.transientRetries)
+	if err != nil {
+		t.Fatal(err)
+	}
+	otherStatus := strings.Replace(privateCase.expectedContent, `"ok"`, `"other"`, 1)
+	for _, tc := range []struct {
+		name            string
+		content         string
+		wantValidJSON   bool
+		wantSchemaMatch *bool
+	}{
+		{name: "invalid JSON", content: "not json", wantValidJSON: false},
+		{name: "schema mismatch", content: otherStatus, wantValidJSON: true, wantSchemaMatch: boolPointer(false)},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			manager, err := capacity.NewManager(nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
+				return &domain.GenerateResponse{Content: tc.content}, nil
+			}))
+			report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !report.Completed || report.Failure != domain.ExecutionFailureValidation ||
+				report.Facts.ValidJSON == nil || *report.Facts.ValidJSON != tc.wantValidJSON ||
+				!sameBool(report.Facts.SchemaMatched, tc.wantSchemaMatch) {
+				t.Fatalf("validation finding = %+v", report)
+			}
+		})
+	}
+}
+
+func TestSmokeTesterConcludesLogicalCapacityExhaustionWithoutDispatch(t *testing.T) {
+	runner := smokeTesterRunner(t, smokeSelectionConfig())
+	manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
+		"alpha": {ConcurrencyLimit: 1},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	release, err := manager.Admit(context.Background(), "alpha")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer release()
+	calls := 0
+	runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
+		calls++
+		return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
+	}))
+	runner.admitter = manager
+
+	report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureCapacity {
+		t.Fatalf("capacity report = %+v", report)
+	}
+	if calls != 0 || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) ||
+		!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
+		t.Fatalf("capacity finding dispatched or invented facts: %+v calls=%d", report, calls)
+	}
+}
+
+func TestSmokeTesterRetainsGenerationEvidenceWhenCallerCancelsBeforeEvaluation(t *testing.T) {
+	runner := smokeTesterRunner(t, smokeSelectionConfig())
+	manager, err := capacity.NewManager(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
+		cancel()
+		return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
+	}))
+
+	report, err := NewSmokeTester(runner, nil).Run(ctx, domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
+	})
+	if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
+		t.Fatalf("partial report=%+v error=%v", report, err)
+	}
+	if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
+		report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
+		report.Facts.TextReturned != nil || report.Facts.ExpectedTextMatched != nil || len(report.Attempts) != 1 {
+		t.Fatalf("cancellation evidence = %+v", report)
+	}
+}
+
+func TestSmokeTesterUsesZeroDefaultRetriesDespiteRunnerDefault(t *testing.T) {
+	runner := smokeTesterRunner(t, smokeSelectionConfig())
+	manager, err := capacity.NewManager(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	runner.llm = generation.NewBuiltIn(manager, reachabilityClient(t, func(*http.Request) (*http.Response, error) {
+		return &http.Response{
+			StatusCode: http.StatusServiceUnavailable,
+			Header:     make(http.Header),
+			Body:       ioNopCloser("retry must not be inherited"),
+		}, nil
+	}))
+	runner.defaultTransientRetries = 5
+
+	report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !report.Completed || report.Failure != domain.ExecutionFailureProviderStatus || len(report.Attempts) != 1 ||
+		report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable {
+		t.Fatalf("retry-isolation report = %+v", report)
+	}
+}
+
+func TestSmokeTesterAggregatesGenerationEvidenceWithoutReplacingLastDispatchFacts(t *testing.T) {
+	status := http.StatusServiceUnavailable
+	requested := true
+	firstStart := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
+	secondStart := firstStart.Add(10 * time.Millisecond)
+	usage := domain.ObservedTokenUsage{
+		PromptTokens:     domain.ObservedTokenCount{Value: 4, Known: true, Complete: false},
+		CompletionTokens: domain.ObservedTokenCount{Value: 9, Known: true, Complete: true},
+	}
+	report := &domain.SmokeTestReport{Target: domain.SmokeTestTarget{Endpoint: "https://alpha.example/v1/chat/completions"}}
+	tester := &SmokeTester{}
+	tester.recordGenerationEvidence(report, generation.GenerationEvidence{
+		Attempts: []generation.AttemptEvidence{
+			{
+				StartedAt: firstStart, Dispatched: true, ResponseReceived: true,
+				HTTP: llm.HTTPExecutionEvidence{
+					DispatchInvoked: true, RequestObserved: true, Endpoint: report.Target.Endpoint,
+					CredentialsSupplied: true, ResponseReceived: true, HTTPStatus: &status,
+				},
+				CapacityWait: 2 * time.Millisecond, RequestDuration: 5 * time.Millisecond,
+				Failure: domain.ExecutionFailureProviderStatus, Usage: usage,
+				StructuredOutputRequested: &requested,
+			},
+			{
+				StartedAt: secondStart, CapacityWait: 7 * time.Millisecond,
+				Failure: domain.ExecutionFailureCanceled,
+			},
+		},
+		RetryDelays: []time.Duration{3 * time.Millisecond},
+		Usage:       usage,
+	})
+
+	if len(report.Attempts) != 2 || !report.Attempts[0].Dispatched || report.Attempts[1].Dispatched ||
+		report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != status ||
+		!report.Attempts[0].Timing.StartedAt.Equal(firstStart) || !report.Attempts[1].Timing.StartedAt.Equal(secondStart) {
+		t.Fatalf("attempt evidence = %+v", report.Attempts)
+	}
+	if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
+		report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
+		report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied ||
+		report.Facts.GenerationCompleted == nil || *report.Facts.GenerationCompleted ||
+		report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested {
+		t.Fatalf("last dispatched facts were replaced: %+v", report.Facts)
+	}
+	if report.Timing.CapacityWait == nil || *report.Timing.CapacityWait != 9*time.Millisecond ||
+		report.Timing.RequestDuration == nil || *report.Timing.RequestDuration != 5*time.Millisecond ||
+		report.Timing.RetryDelay == nil || *report.Timing.RetryDelay != 3*time.Millisecond ||
+		report.Usage != usage {
+		t.Fatalf("aggregated timing/usage = %+v usage=%+v", report.Timing, report.Usage)
+	}
+}
+
+type smokeInferenceClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
+
+func (f smokeInferenceClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	return f(ctx, request)
+}

internal/usecase/smoke_tester.go

@@ -2,6 +2,7 @@ package usecase
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -71,7 +72,10 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
 	if err != nil {
 		return nil, err
 	}
-	if err := t.setSupport(report, selection); err != nil {
+	if isInferenceSmokeTest(request.Kind) {
+		report.Supported = true
+		report.Applicable = boolPointer(true)
+	} else if err := t.setSupport(report, selection); err != nil {
 		return nil, err
 	}
 	if contextErr := ctx.Err(); contextErr != nil {
@@ -83,9 +87,11 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
 		return finish(nil)
 	}
 
-	requestStartedAt := now().UTC()
 	switch request.Kind {
+	case domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput:
+		return t.runInference(ctx, request, selection, report, now, finish)
 	case domain.SmokeTestReachability:
+		requestStartedAt := now().UTC()
 		outcome, err := (reachabilityProbe{client: t.httpClient}).run(
 			ctx,
 			selection.target.Endpoint,
@@ -110,6 +116,7 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
 		t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil)
 		return t.finishHTTP(ctx, report, finish)
 	case domain.SmokeTestAuthentication:
+		requestStartedAt := now().UTC()
 		outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target)
 		requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
 		if err != nil {
@@ -132,6 +139,195 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
 	}
 }
 
+type smokeEvidenceGenerator interface {
+	GenerateWithEvidence(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error, generation.GenerationEvidence)
+}
+
+func (t *SmokeTester) runInference(
+	ctx context.Context,
+	request domain.SmokeTestRequest,
+	selection *smokeTestSelection,
+	report *domain.SmokeTestReport,
+	now func() time.Time,
+	finish func(error) (*domain.SmokeTestReport, error),
+) (*domain.SmokeTestReport, error) {
+	settings := selection.target.EffectiveSettings
+	if err := validateAPIKey(settings.APIKeyEnv, settings.APIKey, settings.APIKeyRequired); err != nil {
+		if contextErr := ctx.Err(); contextErr != nil {
+			report.Failure = generation.ClassifyFailure(contextErr)
+			return finish(contextErr)
+		}
+		if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) {
+			report.Reason = domain.SmokeTestReasonRequiredCredentialMissing
+			report.Failure = domain.ExecutionFailureInvalidRequest
+			report.Usage = noDispatchUsage()
+			report.Completed = true
+			return finish(nil)
+		}
+		return nil, err
+	}
+
+	inferenceCase, err := prepareSmokeInferenceCase(ctx, request.Kind, selection.target, selection.transientRetries)
+	if err != nil {
+		if contextErr := ctx.Err(); contextErr != nil {
+			report.Failure = generation.ClassifyFailure(contextErr)
+			return finish(contextErr)
+		}
+		return nil, err
+	}
+	generator, ok := t.runner.llm.(smokeEvidenceGenerator)
+	if !ok {
+		return nil, fmt.Errorf("smoke-test generation evidence is unavailable")
+	}
+
+	release, err := t.runner.admitRun(ctx, selection.target.BackendID)
+	if err != nil {
+		if contextErr := ctx.Err(); contextErr != nil {
+			report.Failure = generation.ClassifyFailure(contextErr)
+			report.Usage = noDispatchUsage()
+			return finish(contextErr)
+		}
+		var capacityErr *CapacityError
+		if errors.As(err, &capacityErr) {
+			report.Failure = domain.ExecutionFailureCapacity
+			report.Usage = noDispatchUsage()
+			report.Completed = true
+			return finish(nil)
+		}
+		return nil, fmt.Errorf("admit smoke-test generation: %w", err)
+	}
+	defer release()
+
+	response, generationErr, evidence := generator.GenerateWithEvidence(ctx, inferenceCase.request)
+	t.recordGenerationEvidence(report, evidence)
+	if contextErr := ctx.Err(); contextErr != nil {
+		report.Failure = generation.ClassifyFailure(contextErr)
+		return finish(contextErr)
+	}
+	if generationErr != nil {
+		report.Failure = generation.ClassifyFailure(generationErr)
+		report.Completed = true
+		return finish(nil)
+	}
+	if response == nil {
+		report.Failure = generation.ClassifyFailure(llm.ErrMalformedResponse)
+		report.Completed = true
+		return finish(nil)
+	}
+
+	evaluationStartedAt := now().UTC()
+	evaluatedFacts, evaluationErr := inferenceCase.evaluate(ctx, response)
+	evaluationDuration := nonnegativeDuration(now().UTC().Sub(evaluationStartedAt))
+	report.Timing.EvaluationDuration = &evaluationDuration
+	mergeEvaluatedSmokeFacts(&report.Facts, evaluatedFacts)
+	if contextErr := ctx.Err(); contextErr != nil {
+		report.Failure = generation.ClassifyFailure(contextErr)
+		return finish(contextErr)
+	}
+	if evaluationErr != nil {
+		return nil, fmt.Errorf("evaluate smoke-test response: %w", evaluationErr)
+	}
+	report.Completed = true
+	report.Failure = domain.ExecutionFailureNone
+	if request.Kind == domain.SmokeTestStructuredOutput &&
+		((report.Facts.ValidJSON != nil && !*report.Facts.ValidJSON) ||
+			(report.Facts.SchemaMatched != nil && !*report.Facts.SchemaMatched)) {
+		report.Failure = domain.ExecutionFailureValidation
+	}
+	return finish(nil)
+}
+
+func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
+	report.Usage = evidence.Usage
+	var capacityWait, requestDuration time.Duration
+	for _, attemptEvidence := range evidence.Attempts {
+		capacityWait += attemptEvidence.CapacityWait
+		attempt := domain.SmokeTestAttempt{
+			Dispatched:      attemptEvidence.Dispatched,
+			InitialEndpoint: report.Target.Endpoint,
+			Endpoint:        attemptEvidence.HTTP.Endpoint,
+			Facts:           generationAttemptFacts(attemptEvidence),
+			Usage:           attemptEvidence.Usage,
+			Failure:         attemptEvidence.Failure,
+			Timing: domain.SmokeTestTiming{
+				StartedAt:    attemptEvidence.StartedAt,
+				Duration:     nonnegativeDuration(attemptEvidence.CapacityWait + attemptEvidence.RequestDuration),
+				CapacityWait: durationPointer(attemptEvidence.CapacityWait),
+			},
+		}
+		if attemptEvidence.Dispatched {
+			requestDuration += attemptEvidence.RequestDuration
+			attempt.Timing.RequestDuration = durationPointer(attemptEvidence.RequestDuration)
+		}
+		if attemptEvidence.HTTP.HTTPStatus != nil {
+			status := *attemptEvidence.HTTP.HTTPStatus
+			attempt.HTTPStatus = &status
+		}
+		report.Attempts = append(report.Attempts, attempt)
+		if attemptEvidence.Dispatched {
+			report.Facts = generationAttemptFacts(attemptEvidence)
+		}
+	}
+	if len(evidence.Attempts) > 0 {
+		report.Timing.CapacityWait = durationPointer(capacityWait)
+	}
+	if requestDurationKnown(evidence.Attempts) {
+		report.Timing.RequestDuration = durationPointer(requestDuration)
+	}
+	if len(evidence.RetryDelays) > 0 {
+		var retryDelay time.Duration
+		for _, delay := range evidence.RetryDelays {
+			retryDelay += delay
+		}
+		report.Timing.RetryDelay = durationPointer(retryDelay)
+	}
+}
+
+func generationAttemptFacts(evidence generation.AttemptEvidence) domain.SmokeTestFacts {
+	facts := httpExecutionFacts(evidence.HTTP)
+	if evidence.Dispatched {
+		responseReceived := evidence.ResponseReceived
+		generationCompleted := evidence.GenerationCompleted
+		facts.ResponseReceived = &responseReceived
+		facts.GenerationCompleted = &generationCompleted
+	}
+	facts.StructuredOutputRequested = cloneOptionalBool(evidence.StructuredOutputRequested)
+	return facts
+}
+
+func mergeEvaluatedSmokeFacts(target *domain.SmokeTestFacts, evaluated domain.SmokeTestFacts) {
+	if evaluated.TextReturned != nil {
+		target.TextReturned = cloneOptionalBool(evaluated.TextReturned)
+	}
+	if evaluated.ExpectedTextMatched != nil {
+		target.ExpectedTextMatched = cloneOptionalBool(evaluated.ExpectedTextMatched)
+	}
+	if evaluated.ValidJSON != nil {
+		target.ValidJSON = cloneOptionalBool(evaluated.ValidJSON)
+	}
+	if evaluated.SchemaMatched != nil {
+		target.SchemaMatched = cloneOptionalBool(evaluated.SchemaMatched)
+	}
+}
+
+func noDispatchUsage() domain.ObservedTokenUsage {
+	return (observation.UsageAccumulator{}).Observed()
+}
+
+func durationPointer(value time.Duration) *time.Duration {
+	cloned := value
+	return &cloned
+}
+
+func requestDurationKnown(attempts []generation.AttemptEvidence) bool {
+	for _, attempt := range attempts {
+		if attempt.Dispatched {
+			return true
+		}
+	}
+	return false
+}
+
 func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error {
 	switch report.Kind {
 	case domain.SmokeTestReachability: