From b268c55783d1fc3541379471fad377183c6b7cee Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 13 Oct 2026 18:09:20 +0000 Subject: [PATCH] Integrate inference smoke diagnostics --- docs/roadmap/implementation.md | 8 + internal/engine/engine.go | 24 +- internal/engine/smoke_test.go | 101 ++++++++ internal/generation/coordinator.go | 15 +- internal/generation/evidence.go | 1 + internal/usecase/smoke_inference_test.go | 288 +++++++++++++++++++++++ internal/usecase/smoke_tester.go | 200 +++++++++++++++- 7 files changed, 624 insertions(+), 13 deletions(-) create mode 100644 internal/engine/smoke_test.go create mode 100644 internal/usecase/smoke_inference_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 35366f6..7b7fdc0 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -622,6 +622,14 @@ Validate: focused use-case/engine tests for text/schema results, retry-default isolation, injected generation, required-credential/capacity findings, and evidence after cancellation. Stage 17 owns broader facade-level capacity/observer regressions. +**Completed: 2026-10-13.** Integrated private inference cases through the shared +generation coordinator, preserving attempts, usage, phase timings, last-dispatched +facts, and local validation findings. Required-credential and logical-capacity +findings conclude without dispatch. Built-in and injected engines share the +runner/coordinator while only built-in engines supply HTTP diagnostics. +Validation passed: `go test -race ./internal/usecase ./internal/generation +./internal/engine`, `go vet ./...`, `go test ./...`, and `git diff --check`. + ## Stage 16: Expose The Public SmokeTest Facade Scope: root declarations, GoDoc, conversions, and external-package API tests. diff --git a/internal/engine/engine.go b/internal/engine/engine.go index f27084a..d1d56da 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -81,9 +81,10 @@ func (o *Options) SetSchemaFile(path string) error { // Engine forwards domain operations to the single assembled use-case runner. type Engine struct { - runner *usecase.Runner - inspection configuration.Inspection - listing identityInventory + runner *usecase.Runner + smokeTester *usecase.SmokeTester + inspection configuration.Inspection + listing identityInventory } type identityInventory interface { @@ -124,11 +125,13 @@ func New(cfg Config, options Options) (*Engine, error) { validator = validate.NewStandardValidator(schemaDir) } var client llm.Client + var diagnosticHTTPClient *llm.OpenAICompatibleClient if options.llmClient == nil { builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient}) if newErr != nil { return nil, newErr } + diagnosticHTTPClient = builtIn client = generation.NewBuiltIn(manager, builtIn) } else { client = generation.NewInjected(manager, options.llmClient) @@ -138,10 +141,12 @@ func New(cfg Config, options Options) (*Engine, error) { reader = artifact.NewCompositeReader() } + runner := usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries) return &Engine{ - runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries), - inspection: snapshot.Inspection(), - listing: snapshot, + runner: runner, + smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient), + inspection: snapshot.Inspection(), + listing: snapshot, }, nil } @@ -214,3 +219,10 @@ func (e *Engine) Run(ctx context.Context, request domain.RunRequest) (*domain.Ru func (e *Engine) RunPrepared(ctx context.Context, prepared *usecase.PreparedExecution) (*domain.RunResult, error) { return e.runner.RunPrepared(ctx, prepared) } + +func (e *Engine) SmokeTest(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) { + if e == nil || e.smokeTester == nil { + return nil, fmt.Errorf("%w: smoke tester is not configured", usecase.ErrInvalidRequest) + } + return e.smokeTester.Run(ctx, request) +} diff --git a/internal/engine/smoke_test.go b/internal/engine/smoke_test.go new file mode 100644 index 0000000..b4e4bfe --- /dev/null +++ b/internal/engine/smoke_test.go @@ -0,0 +1,101 @@ +package engine + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { + calls := 0 + options := Options{} + options.SetLLMClient(engineSmokeClientFunc(func(_ context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) { + calls++ + if len(request.Prompt.Messages) != 1 || request.Prompt.Messages[0].Role != "user" { + t.Fatalf("injected generation request = %+v", request) + } + return &domain.GenerateResponse{Content: "engine response"}, nil + })) + assembled, err := New(Config{}, options) + if err != nil { + t.Fatal(err) + } + + report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome", + }) + if err != nil { + t.Fatal(err) + } + if calls != 1 || !report.Completed || !report.Supported || report.Applicable == nil || !*report.Applicable || + report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || + report.Facts.TextReturned == nil || !*report.Facts.TextReturned || + report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || + report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" { + t.Fatalf("injected smoke report calls=%d: %+v", calls, report) + } + + authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome", + }) + if err != nil { + t.Fatal(err) + } + if !authentication.Supported || authentication.Applicable == nil || *authentication.Applicable || + authentication.Reason != domain.SmokeTestReasonAuthenticationNotApplicable || len(authentication.Attempts) != 0 { + t.Fatalf("injected authentication report = %+v", authentication) + } +} + +func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) { + var observedURL string + var authorization string + options := Options{} + assembled, err := New(Config{HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) { + observedURL = request.URL.String() + authorization = request.Header.Get("Authorization") + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"engine response"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)), + }, nil + })}}, options) + if err != nil { + t.Fatal(err) + } + + report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome", + }) + if err != nil { + t.Fatal(err) + } + if observedURL == "" || authorization != "" { + t.Fatalf("built-in request URL=%q authorization=%q", observedURL, authorization) + } + if !report.Completed || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || + report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || + report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || + report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied || + report.Facts.TextReturned == nil || !*report.Facts.TextReturned || + report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || len(report.Attempts) != 1 || + report.Attempts[0].Endpoint != observedURL { + t.Fatalf("built-in smoke report = %+v", report) + } +} + +type engineSmokeClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) + +func (f engineSmokeClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) { + return f(ctx, request) +} + +type engineRoundTripperFunc func(*http.Request) (*http.Response, error) + +func (f engineRoundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/generation/coordinator.go b/internal/generation/coordinator.go index 2d0b8ee..aff2f12 100644 --- a/internal/generation/coordinator.go +++ b/internal/generation/coordinator.go @@ -117,12 +117,13 @@ func (c *Coordinator) generate( return finish(nil, err) } ordinal := attempts + 1 - outcome, dispatched, capacityWait, generationDuration := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence) + outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence) if dispatched { attempts++ } if captureEvidence { attempt := AttemptEvidence{ + StartedAt: startedAt.UTC(), Dispatched: dispatched, ResponseReceived: outcome.Evidence.ResponseReceived, HTTP: cloneHTTPEvidence(outcome.Evidence), @@ -220,6 +221,7 @@ func (c *Coordinator) generateInjected( if err != nil { if captureEvidence { evidence.Attempts = append(evidence.Attempts, AttemptEvidence{ + StartedAt: start.UTC(), CapacityWait: capacityWait, Failure: ClassifyFailure(err), }) @@ -234,6 +236,7 @@ func (c *Coordinator) generateInjected( release() if captureEvidence { evidence.Attempts = append(evidence.Attempts, AttemptEvidence{ + StartedAt: start.UTC(), CapacityWait: capacityWait, Failure: ClassifyFailure(err), }) @@ -251,6 +254,7 @@ func (c *Coordinator) generateInjected( } if captureEvidence { attempt := AttemptEvidence{ + StartedAt: start.UTC(), Dispatched: true, ResponseReceived: response != nil, RequestDuration: duration, @@ -293,16 +297,17 @@ func (c *Coordinator) generateInjected( return finish(response, nil) } -func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration) { +func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) { start := c.now() + startedAt = start release, err := c.manager.Acquire(ctx, backendID) capacityWait = c.capacityWait(backendID, start) if err != nil { - return llm.AttemptOutcome{Err: err}, false, capacityWait, 0 + return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt } defer release() if err := ctx.Err(); err != nil { - return llm.AttemptOutcome{Err: err}, false, capacityWait, 0 + return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt } dispatchStart := c.now() if captureEvidence { @@ -312,7 +317,7 @@ func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *l outcome = prepared.Execute(ctx) dispatched = true } - return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()) + return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt } func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration { diff --git a/internal/generation/evidence.go b/internal/generation/evidence.go index 134237c..60ab891 100644 --- a/internal/generation/evidence.go +++ b/internal/generation/evidence.go @@ -18,6 +18,7 @@ type GenerationEvidence struct { // AttemptEvidence describes one capacity acquisition and any dispatch it // reached. Generated content remains on the separate response return value. type AttemptEvidence struct { + StartedAt time.Time Dispatched bool ResponseReceived bool HTTP llm.HTTPExecutionEvidence diff --git a/internal/usecase/smoke_inference_test.go b/internal/usecase/smoke_inference_test.go new file mode 100644 index 0000000..7cd9852 --- /dev/null +++ b/internal/usecase/smoke_inference_test.go @@ -0,0 +1,288 @@ +package usecase + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/promptkit/internal/capacity" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/generation" + "gitea.maximumdirect.net/eric/promptkit/internal/llm" +) + +func TestSmokeTesterRunsPrivateInferenceCasesThroughInjectedCoordinator(t *testing.T) { + for _, kind := range []domain.SmokeTestKind{domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput} { + t.Run(string(kind), func(t *testing.T) { + runner := smokeTesterRunner(t, smokeSelectionConfig()) + request := domain.SmokeTestRequest{Kind: kind, ProfileID: "alpha-profile"} + selection, err := runner.resolveSmokeTestTarget(context.Background(), request) + if err != nil { + t.Fatal(err) + } + privateCase, err := prepareSmokeInferenceCase(context.Background(), kind, selection.target, selection.transientRetries) + if err != nil { + t.Fatal(err) + } + calls := 0 + client := smokeInferenceClientFunc(func(_ context.Context, generationRequest domain.GenerateRequest) (*domain.GenerateResponse, error) { + calls++ + if generationRequest.TransientRetries != 0 || generationRequest.Generation != 1 || generationRequest.Target.BackendID != "alpha" { + t.Fatalf("generation request selection/retries = %+v", generationRequest) + } + return &domain.GenerateResponse{Content: privateCase.expectedContent}, nil + }) + manager, err := capacity.NewManager(nil) + if err != nil { + t.Fatal(err) + } + runner.llm = generation.NewInjected(manager, client) + runner.admitter = &smokeAdmissionSpy{} + runner.defaultTransientRetries = 5 + + report, err := NewSmokeTester(runner, nil).Run(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if calls != 1 || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone { + t.Fatalf("calls=%d report=%+v", calls, report) + } + if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || + report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.EndpointReachable != nil { + t.Fatalf("generation facts = %+v", report.Facts) + } + if kind == domain.SmokeTestTextInference { + if report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || !*report.Facts.ExpectedTextMatched { + t.Fatalf("text facts = %+v", report.Facts) + } + } else if report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested || + report.Facts.ValidJSON == nil || !*report.Facts.ValidJSON || report.Facts.SchemaMatched == nil || !*report.Facts.SchemaMatched { + t.Fatalf("structured facts = %+v", report.Facts) + } + if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].Timing.StartedAt.IsZero() || + report.Attempts[0].Timing.CapacityWait == nil || report.Attempts[0].Timing.RequestDuration == nil || + report.Timing.CapacityWait == nil || report.Timing.RequestDuration == nil || report.Timing.EvaluationDuration == nil { + t.Fatalf("attempt/timing evidence = %+v", report) + } + if report.Usage.PromptTokens.Known || report.Attempts[0].Usage.PromptTokens.Known { + t.Fatalf("unexpected usage evidence = %+v", report.Usage) + } + }) + } +} + +func TestSmokeTesterConcludesMissingRequiredCredentialWithoutDispatch(t *testing.T) { + values := smokeSelectionConfig() + profiles := values["profiles"].(map[string]any) + profiles["required-profile"] = map[string]any{ + "backend": "alpha", "model": "required-model", "api_key_required": true, + } + runner := smokeTesterRunner(t, values) + admitter := &smokeAdmissionSpy{} + runner.admitter = admitter + + report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, ProfileID: "required-profile", + }) + if err != nil { + t.Fatal(err) + } + if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || + report.Reason != domain.SmokeTestReasonRequiredCredentialMissing || report.Failure != domain.ExecutionFailureInvalidRequest { + t.Fatalf("missing-credential report = %+v", report) + } + if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || admitter.calls != 0 || + !report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 { + t.Fatalf("missing credential invented dispatch evidence: %+v admission calls=%d", report, admitter.calls) + } +} + +func TestSmokeTesterReportsStructuredContentValidationFindings(t *testing.T) { + runner := smokeTesterRunner(t, smokeSelectionConfig()) + request := domain.SmokeTestRequest{Kind: domain.SmokeTestStructuredOutput, ProfileID: "alpha-profile"} + selection, err := runner.resolveSmokeTestTarget(context.Background(), request) + if err != nil { + t.Fatal(err) + } + privateCase, err := prepareSmokeInferenceCase(context.Background(), request.Kind, selection.target, selection.transientRetries) + if err != nil { + t.Fatal(err) + } + otherStatus := strings.Replace(privateCase.expectedContent, `"ok"`, `"other"`, 1) + for _, tc := range []struct { + name string + content string + wantValidJSON bool + wantSchemaMatch *bool + }{ + {name: "invalid JSON", content: "not json", wantValidJSON: false}, + {name: "schema mismatch", content: otherStatus, wantValidJSON: true, wantSchemaMatch: boolPointer(false)}, + } { + t.Run(tc.name, func(t *testing.T) { + manager, err := capacity.NewManager(nil) + if err != nil { + t.Fatal(err) + } + runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { + return &domain.GenerateResponse{Content: tc.content}, nil + })) + report, err := NewSmokeTester(runner, nil).Run(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if !report.Completed || report.Failure != domain.ExecutionFailureValidation || + report.Facts.ValidJSON == nil || *report.Facts.ValidJSON != tc.wantValidJSON || + !sameBool(report.Facts.SchemaMatched, tc.wantSchemaMatch) { + t.Fatalf("validation finding = %+v", report) + } + }) + } +} + +func TestSmokeTesterConcludesLogicalCapacityExhaustionWithoutDispatch(t *testing.T) { + runner := smokeTesterRunner(t, smokeSelectionConfig()) + manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{ + "alpha": {ConcurrencyLimit: 1}, + }) + if err != nil { + t.Fatal(err) + } + release, err := manager.Admit(context.Background(), "alpha") + if err != nil { + t.Fatal(err) + } + defer release() + calls := 0 + runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { + calls++ + return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil + })) + runner.admitter = manager + + report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", + }) + if err != nil { + t.Fatal(err) + } + if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureCapacity { + t.Fatalf("capacity report = %+v", report) + } + if calls != 0 || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || + !report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 { + t.Fatalf("capacity finding dispatched or invented facts: %+v calls=%d", report, calls) + } +} + +func TestSmokeTesterRetainsGenerationEvidenceWhenCallerCancelsBeforeEvaluation(t *testing.T) { + runner := smokeTesterRunner(t, smokeSelectionConfig()) + manager, err := capacity.NewManager(nil) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { + cancel() + return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil + })) + + report, err := NewSmokeTester(runner, nil).Run(ctx, domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", + }) + if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled { + t.Fatalf("partial report=%+v error=%v", report, err) + } + if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || + report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || + report.Facts.TextReturned != nil || report.Facts.ExpectedTextMatched != nil || len(report.Attempts) != 1 { + t.Fatalf("cancellation evidence = %+v", report) + } +} + +func TestSmokeTesterUsesZeroDefaultRetriesDespiteRunnerDefault(t *testing.T) { + runner := smokeTesterRunner(t, smokeSelectionConfig()) + manager, err := capacity.NewManager(nil) + if err != nil { + t.Fatal(err) + } + runner.llm = generation.NewBuiltIn(manager, reachabilityClient(t, func(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusServiceUnavailable, + Header: make(http.Header), + Body: ioNopCloser("retry must not be inherited"), + }, nil + })) + runner.defaultTransientRetries = 5 + + report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", + }) + if err != nil { + t.Fatal(err) + } + if !report.Completed || report.Failure != domain.ExecutionFailureProviderStatus || len(report.Attempts) != 1 || + report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable { + t.Fatalf("retry-isolation report = %+v", report) + } +} + +func TestSmokeTesterAggregatesGenerationEvidenceWithoutReplacingLastDispatchFacts(t *testing.T) { + status := http.StatusServiceUnavailable + requested := true + firstStart := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) + secondStart := firstStart.Add(10 * time.Millisecond) + usage := domain.ObservedTokenUsage{ + PromptTokens: domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}, + CompletionTokens: domain.ObservedTokenCount{Value: 9, Known: true, Complete: true}, + } + report := &domain.SmokeTestReport{Target: domain.SmokeTestTarget{Endpoint: "https://alpha.example/v1/chat/completions"}} + tester := &SmokeTester{} + tester.recordGenerationEvidence(report, generation.GenerationEvidence{ + Attempts: []generation.AttemptEvidence{ + { + StartedAt: firstStart, Dispatched: true, ResponseReceived: true, + HTTP: llm.HTTPExecutionEvidence{ + DispatchInvoked: true, RequestObserved: true, Endpoint: report.Target.Endpoint, + CredentialsSupplied: true, ResponseReceived: true, HTTPStatus: &status, + }, + CapacityWait: 2 * time.Millisecond, RequestDuration: 5 * time.Millisecond, + Failure: domain.ExecutionFailureProviderStatus, Usage: usage, + StructuredOutputRequested: &requested, + }, + { + StartedAt: secondStart, CapacityWait: 7 * time.Millisecond, + Failure: domain.ExecutionFailureCanceled, + }, + }, + RetryDelays: []time.Duration{3 * time.Millisecond}, + Usage: usage, + }) + + if len(report.Attempts) != 2 || !report.Attempts[0].Dispatched || report.Attempts[1].Dispatched || + report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != status || + !report.Attempts[0].Timing.StartedAt.Equal(firstStart) || !report.Attempts[1].Timing.StartedAt.Equal(secondStart) { + t.Fatalf("attempt evidence = %+v", report.Attempts) + } + if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || + report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || + report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied || + report.Facts.GenerationCompleted == nil || *report.Facts.GenerationCompleted || + report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested { + t.Fatalf("last dispatched facts were replaced: %+v", report.Facts) + } + if report.Timing.CapacityWait == nil || *report.Timing.CapacityWait != 9*time.Millisecond || + report.Timing.RequestDuration == nil || *report.Timing.RequestDuration != 5*time.Millisecond || + report.Timing.RetryDelay == nil || *report.Timing.RetryDelay != 3*time.Millisecond || + report.Usage != usage { + t.Fatalf("aggregated timing/usage = %+v usage=%+v", report.Timing, report.Usage) + } +} + +type smokeInferenceClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) + +func (f smokeInferenceClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) { + return f(ctx, request) +} diff --git a/internal/usecase/smoke_tester.go b/internal/usecase/smoke_tester.go index f7462cb..4310fd6 100644 --- a/internal/usecase/smoke_tester.go +++ b/internal/usecase/smoke_tester.go @@ -2,6 +2,7 @@ package usecase import ( "context" + "errors" "fmt" "time" @@ -71,7 +72,10 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) if err != nil { return nil, err } - if err := t.setSupport(report, selection); err != nil { + if isInferenceSmokeTest(request.Kind) { + report.Supported = true + report.Applicable = boolPointer(true) + } else if err := t.setSupport(report, selection); err != nil { return nil, err } if contextErr := ctx.Err(); contextErr != nil { @@ -83,9 +87,11 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) return finish(nil) } - requestStartedAt := now().UTC() switch request.Kind { + case domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput: + return t.runInference(ctx, request, selection, report, now, finish) case domain.SmokeTestReachability: + requestStartedAt := now().UTC() outcome, err := (reachabilityProbe{client: t.httpClient}).run( ctx, selection.target.Endpoint, @@ -110,6 +116,7 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil) return t.finishHTTP(ctx, report, finish) case domain.SmokeTestAuthentication: + requestStartedAt := now().UTC() outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target) requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt)) if err != nil { @@ -132,6 +139,195 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) } } +type smokeEvidenceGenerator interface { + GenerateWithEvidence(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error, generation.GenerationEvidence) +} + +func (t *SmokeTester) runInference( + ctx context.Context, + request domain.SmokeTestRequest, + selection *smokeTestSelection, + report *domain.SmokeTestReport, + now func() time.Time, + finish func(error) (*domain.SmokeTestReport, error), +) (*domain.SmokeTestReport, error) { + settings := selection.target.EffectiveSettings + if err := validateAPIKey(settings.APIKeyEnv, settings.APIKey, settings.APIKeyRequired); err != nil { + if contextErr := ctx.Err(); contextErr != nil { + report.Failure = generation.ClassifyFailure(contextErr) + return finish(contextErr) + } + if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) { + report.Reason = domain.SmokeTestReasonRequiredCredentialMissing + report.Failure = domain.ExecutionFailureInvalidRequest + report.Usage = noDispatchUsage() + report.Completed = true + return finish(nil) + } + return nil, err + } + + inferenceCase, err := prepareSmokeInferenceCase(ctx, request.Kind, selection.target, selection.transientRetries) + if err != nil { + if contextErr := ctx.Err(); contextErr != nil { + report.Failure = generation.ClassifyFailure(contextErr) + return finish(contextErr) + } + return nil, err + } + generator, ok := t.runner.llm.(smokeEvidenceGenerator) + if !ok { + return nil, fmt.Errorf("smoke-test generation evidence is unavailable") + } + + release, err := t.runner.admitRun(ctx, selection.target.BackendID) + if err != nil { + if contextErr := ctx.Err(); contextErr != nil { + report.Failure = generation.ClassifyFailure(contextErr) + report.Usage = noDispatchUsage() + return finish(contextErr) + } + var capacityErr *CapacityError + if errors.As(err, &capacityErr) { + report.Failure = domain.ExecutionFailureCapacity + report.Usage = noDispatchUsage() + report.Completed = true + return finish(nil) + } + return nil, fmt.Errorf("admit smoke-test generation: %w", err) + } + defer release() + + response, generationErr, evidence := generator.GenerateWithEvidence(ctx, inferenceCase.request) + t.recordGenerationEvidence(report, evidence) + if contextErr := ctx.Err(); contextErr != nil { + report.Failure = generation.ClassifyFailure(contextErr) + return finish(contextErr) + } + if generationErr != nil { + report.Failure = generation.ClassifyFailure(generationErr) + report.Completed = true + return finish(nil) + } + if response == nil { + report.Failure = generation.ClassifyFailure(llm.ErrMalformedResponse) + report.Completed = true + return finish(nil) + } + + evaluationStartedAt := now().UTC() + evaluatedFacts, evaluationErr := inferenceCase.evaluate(ctx, response) + evaluationDuration := nonnegativeDuration(now().UTC().Sub(evaluationStartedAt)) + report.Timing.EvaluationDuration = &evaluationDuration + mergeEvaluatedSmokeFacts(&report.Facts, evaluatedFacts) + if contextErr := ctx.Err(); contextErr != nil { + report.Failure = generation.ClassifyFailure(contextErr) + return finish(contextErr) + } + if evaluationErr != nil { + return nil, fmt.Errorf("evaluate smoke-test response: %w", evaluationErr) + } + report.Completed = true + report.Failure = domain.ExecutionFailureNone + if request.Kind == domain.SmokeTestStructuredOutput && + ((report.Facts.ValidJSON != nil && !*report.Facts.ValidJSON) || + (report.Facts.SchemaMatched != nil && !*report.Facts.SchemaMatched)) { + report.Failure = domain.ExecutionFailureValidation + } + return finish(nil) +} + +func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) { + report.Usage = evidence.Usage + var capacityWait, requestDuration time.Duration + for _, attemptEvidence := range evidence.Attempts { + capacityWait += attemptEvidence.CapacityWait + attempt := domain.SmokeTestAttempt{ + Dispatched: attemptEvidence.Dispatched, + InitialEndpoint: report.Target.Endpoint, + Endpoint: attemptEvidence.HTTP.Endpoint, + Facts: generationAttemptFacts(attemptEvidence), + Usage: attemptEvidence.Usage, + Failure: attemptEvidence.Failure, + Timing: domain.SmokeTestTiming{ + StartedAt: attemptEvidence.StartedAt, + Duration: nonnegativeDuration(attemptEvidence.CapacityWait + attemptEvidence.RequestDuration), + CapacityWait: durationPointer(attemptEvidence.CapacityWait), + }, + } + if attemptEvidence.Dispatched { + requestDuration += attemptEvidence.RequestDuration + attempt.Timing.RequestDuration = durationPointer(attemptEvidence.RequestDuration) + } + if attemptEvidence.HTTP.HTTPStatus != nil { + status := *attemptEvidence.HTTP.HTTPStatus + attempt.HTTPStatus = &status + } + report.Attempts = append(report.Attempts, attempt) + if attemptEvidence.Dispatched { + report.Facts = generationAttemptFacts(attemptEvidence) + } + } + if len(evidence.Attempts) > 0 { + report.Timing.CapacityWait = durationPointer(capacityWait) + } + if requestDurationKnown(evidence.Attempts) { + report.Timing.RequestDuration = durationPointer(requestDuration) + } + if len(evidence.RetryDelays) > 0 { + var retryDelay time.Duration + for _, delay := range evidence.RetryDelays { + retryDelay += delay + } + report.Timing.RetryDelay = durationPointer(retryDelay) + } +} + +func generationAttemptFacts(evidence generation.AttemptEvidence) domain.SmokeTestFacts { + facts := httpExecutionFacts(evidence.HTTP) + if evidence.Dispatched { + responseReceived := evidence.ResponseReceived + generationCompleted := evidence.GenerationCompleted + facts.ResponseReceived = &responseReceived + facts.GenerationCompleted = &generationCompleted + } + facts.StructuredOutputRequested = cloneOptionalBool(evidence.StructuredOutputRequested) + return facts +} + +func mergeEvaluatedSmokeFacts(target *domain.SmokeTestFacts, evaluated domain.SmokeTestFacts) { + if evaluated.TextReturned != nil { + target.TextReturned = cloneOptionalBool(evaluated.TextReturned) + } + if evaluated.ExpectedTextMatched != nil { + target.ExpectedTextMatched = cloneOptionalBool(evaluated.ExpectedTextMatched) + } + if evaluated.ValidJSON != nil { + target.ValidJSON = cloneOptionalBool(evaluated.ValidJSON) + } + if evaluated.SchemaMatched != nil { + target.SchemaMatched = cloneOptionalBool(evaluated.SchemaMatched) + } +} + +func noDispatchUsage() domain.ObservedTokenUsage { + return (observation.UsageAccumulator{}).Observed() +} + +func durationPointer(value time.Duration) *time.Duration { + cloned := value + return &cloned +} + +func requestDurationKnown(attempts []generation.AttemptEvidence) bool { + for _, attempt := range attempts { + if attempt.Dispatched { + return true + } + } + return false +} + func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error { switch report.Kind { case domain.SmokeTestReachability: