Integrate inference smoke diagnostics
This commit is contained in:
288
internal/usecase/smoke_inference_test.go
Normal file
288
internal/usecase/smoke_inference_test.go
Normal file
@@ -0,0 +1,288 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
func TestSmokeTesterRunsPrivateInferenceCasesThroughInjectedCoordinator(t *testing.T) {
|
||||
for _, kind := range []domain.SmokeTestKind{domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput} {
|
||||
t.Run(string(kind), func(t *testing.T) {
|
||||
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
||||
request := domain.SmokeTestRequest{Kind: kind, ProfileID: "alpha-profile"}
|
||||
selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
privateCase, err := prepareSmokeInferenceCase(context.Background(), kind, selection.target, selection.transientRetries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
client := smokeInferenceClientFunc(func(_ context.Context, generationRequest domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
calls++
|
||||
if generationRequest.TransientRetries != 0 || generationRequest.Generation != 1 || generationRequest.Target.BackendID != "alpha" {
|
||||
t.Fatalf("generation request selection/retries = %+v", generationRequest)
|
||||
}
|
||||
return &domain.GenerateResponse{Content: privateCase.expectedContent}, nil
|
||||
})
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.llm = generation.NewInjected(manager, client)
|
||||
runner.admitter = &smokeAdmissionSpy{}
|
||||
runner.defaultTransientRetries = 5
|
||||
|
||||
report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone {
|
||||
t.Fatalf("calls=%d report=%+v", calls, report)
|
||||
}
|
||||
if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
||||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.EndpointReachable != nil {
|
||||
t.Fatalf("generation facts = %+v", report.Facts)
|
||||
}
|
||||
if kind == domain.SmokeTestTextInference {
|
||||
if report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || !*report.Facts.ExpectedTextMatched {
|
||||
t.Fatalf("text facts = %+v", report.Facts)
|
||||
}
|
||||
} else if report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested ||
|
||||
report.Facts.ValidJSON == nil || !*report.Facts.ValidJSON || report.Facts.SchemaMatched == nil || !*report.Facts.SchemaMatched {
|
||||
t.Fatalf("structured facts = %+v", report.Facts)
|
||||
}
|
||||
if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].Timing.StartedAt.IsZero() ||
|
||||
report.Attempts[0].Timing.CapacityWait == nil || report.Attempts[0].Timing.RequestDuration == nil ||
|
||||
report.Timing.CapacityWait == nil || report.Timing.RequestDuration == nil || report.Timing.EvaluationDuration == nil {
|
||||
t.Fatalf("attempt/timing evidence = %+v", report)
|
||||
}
|
||||
if report.Usage.PromptTokens.Known || report.Attempts[0].Usage.PromptTokens.Known {
|
||||
t.Fatalf("unexpected usage evidence = %+v", report.Usage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterConcludesMissingRequiredCredentialWithoutDispatch(t *testing.T) {
|
||||
values := smokeSelectionConfig()
|
||||
profiles := values["profiles"].(map[string]any)
|
||||
profiles["required-profile"] = map[string]any{
|
||||
"backend": "alpha", "model": "required-model", "api_key_required": true,
|
||||
}
|
||||
runner := smokeTesterRunner(t, values)
|
||||
admitter := &smokeAdmissionSpy{}
|
||||
runner.admitter = admitter
|
||||
|
||||
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, ProfileID: "required-profile",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed ||
|
||||
report.Reason != domain.SmokeTestReasonRequiredCredentialMissing || report.Failure != domain.ExecutionFailureInvalidRequest {
|
||||
t.Fatalf("missing-credential report = %+v", report)
|
||||
}
|
||||
if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || admitter.calls != 0 ||
|
||||
!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
|
||||
t.Fatalf("missing credential invented dispatch evidence: %+v admission calls=%d", report, admitter.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterReportsStructuredContentValidationFindings(t *testing.T) {
|
||||
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
||||
request := domain.SmokeTestRequest{Kind: domain.SmokeTestStructuredOutput, ProfileID: "alpha-profile"}
|
||||
selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
privateCase, err := prepareSmokeInferenceCase(context.Background(), request.Kind, selection.target, selection.transientRetries)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherStatus := strings.Replace(privateCase.expectedContent, `"ok"`, `"other"`, 1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
content string
|
||||
wantValidJSON bool
|
||||
wantSchemaMatch *bool
|
||||
}{
|
||||
{name: "invalid JSON", content: "not json", wantValidJSON: false},
|
||||
{name: "schema mismatch", content: otherStatus, wantValidJSON: true, wantSchemaMatch: boolPointer(false)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return &domain.GenerateResponse{Content: tc.content}, nil
|
||||
}))
|
||||
report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.Completed || report.Failure != domain.ExecutionFailureValidation ||
|
||||
report.Facts.ValidJSON == nil || *report.Facts.ValidJSON != tc.wantValidJSON ||
|
||||
!sameBool(report.Facts.SchemaMatched, tc.wantSchemaMatch) {
|
||||
t.Fatalf("validation finding = %+v", report)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterConcludesLogicalCapacityExhaustionWithoutDispatch(t *testing.T) {
|
||||
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
||||
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
||||
"alpha": {ConcurrencyLimit: 1},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release, err := manager.Admit(context.Background(), "alpha")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
calls := 0
|
||||
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
calls++
|
||||
return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
|
||||
}))
|
||||
runner.admitter = manager
|
||||
|
||||
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureCapacity {
|
||||
t.Fatalf("capacity report = %+v", report)
|
||||
}
|
||||
if calls != 0 || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) ||
|
||||
!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
|
||||
t.Fatalf("capacity finding dispatched or invented facts: %+v calls=%d", report, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterRetainsGenerationEvidenceWhenCallerCancelsBeforeEvaluation(t *testing.T) {
|
||||
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
cancel()
|
||||
return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
|
||||
}))
|
||||
|
||||
report, err := NewSmokeTester(runner, nil).Run(ctx, domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
|
||||
t.Fatalf("partial report=%+v error=%v", report, err)
|
||||
}
|
||||
if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
||||
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
||||
report.Facts.TextReturned != nil || report.Facts.ExpectedTextMatched != nil || len(report.Attempts) != 1 {
|
||||
t.Fatalf("cancellation evidence = %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterUsesZeroDefaultRetriesDespiteRunnerDefault(t *testing.T) {
|
||||
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
||||
manager, err := capacity.NewManager(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.llm = generation.NewBuiltIn(manager, reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Header: make(http.Header),
|
||||
Body: ioNopCloser("retry must not be inherited"),
|
||||
}, nil
|
||||
}))
|
||||
runner.defaultTransientRetries = 5
|
||||
|
||||
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
||||
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.Completed || report.Failure != domain.ExecutionFailureProviderStatus || len(report.Attempts) != 1 ||
|
||||
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable {
|
||||
t.Fatalf("retry-isolation report = %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTesterAggregatesGenerationEvidenceWithoutReplacingLastDispatchFacts(t *testing.T) {
|
||||
status := http.StatusServiceUnavailable
|
||||
requested := true
|
||||
firstStart := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
||||
secondStart := firstStart.Add(10 * time.Millisecond)
|
||||
usage := domain.ObservedTokenUsage{
|
||||
PromptTokens: domain.ObservedTokenCount{Value: 4, Known: true, Complete: false},
|
||||
CompletionTokens: domain.ObservedTokenCount{Value: 9, Known: true, Complete: true},
|
||||
}
|
||||
report := &domain.SmokeTestReport{Target: domain.SmokeTestTarget{Endpoint: "https://alpha.example/v1/chat/completions"}}
|
||||
tester := &SmokeTester{}
|
||||
tester.recordGenerationEvidence(report, generation.GenerationEvidence{
|
||||
Attempts: []generation.AttemptEvidence{
|
||||
{
|
||||
StartedAt: firstStart, Dispatched: true, ResponseReceived: true,
|
||||
HTTP: llm.HTTPExecutionEvidence{
|
||||
DispatchInvoked: true, RequestObserved: true, Endpoint: report.Target.Endpoint,
|
||||
CredentialsSupplied: true, ResponseReceived: true, HTTPStatus: &status,
|
||||
},
|
||||
CapacityWait: 2 * time.Millisecond, RequestDuration: 5 * time.Millisecond,
|
||||
Failure: domain.ExecutionFailureProviderStatus, Usage: usage,
|
||||
StructuredOutputRequested: &requested,
|
||||
},
|
||||
{
|
||||
StartedAt: secondStart, CapacityWait: 7 * time.Millisecond,
|
||||
Failure: domain.ExecutionFailureCanceled,
|
||||
},
|
||||
},
|
||||
RetryDelays: []time.Duration{3 * time.Millisecond},
|
||||
Usage: usage,
|
||||
})
|
||||
|
||||
if len(report.Attempts) != 2 || !report.Attempts[0].Dispatched || report.Attempts[1].Dispatched ||
|
||||
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != status ||
|
||||
!report.Attempts[0].Timing.StartedAt.Equal(firstStart) || !report.Attempts[1].Timing.StartedAt.Equal(secondStart) {
|
||||
t.Fatalf("attempt evidence = %+v", report.Attempts)
|
||||
}
|
||||
if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
||||
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
||||
report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied ||
|
||||
report.Facts.GenerationCompleted == nil || *report.Facts.GenerationCompleted ||
|
||||
report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested {
|
||||
t.Fatalf("last dispatched facts were replaced: %+v", report.Facts)
|
||||
}
|
||||
if report.Timing.CapacityWait == nil || *report.Timing.CapacityWait != 9*time.Millisecond ||
|
||||
report.Timing.RequestDuration == nil || *report.Timing.RequestDuration != 5*time.Millisecond ||
|
||||
report.Timing.RetryDelay == nil || *report.Timing.RetryDelay != 3*time.Millisecond ||
|
||||
report.Usage != usage {
|
||||
t.Fatalf("aggregated timing/usage = %+v usage=%+v", report.Timing, report.Usage)
|
||||
}
|
||||
}
|
||||
|
||||
type smokeInferenceClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
|
||||
|
||||
func (f smokeInferenceClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
return f(ctx, request)
|
||||
}
|
||||
Reference in New Issue
Block a user