package usecase import ( "context" "errors" "net/http" "strings" "testing" "time" "gitea.maximumdirect.net/eric/promptkit/internal/capacity" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/generation" "gitea.maximumdirect.net/eric/promptkit/internal/llm" ) func TestSmokeTesterRunsPrivateInferenceCasesThroughInjectedCoordinator(t *testing.T) { for _, kind := range []domain.SmokeTestKind{domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput} { t.Run(string(kind), func(t *testing.T) { runner := smokeTesterRunner(t, smokeSelectionConfig()) request := domain.SmokeTestRequest{Kind: kind, ProfileID: "alpha-profile"} selection, err := runner.resolveSmokeTestTarget(context.Background(), request) if err != nil { t.Fatal(err) } privateCase, err := prepareSmokeInferenceCase(context.Background(), kind, selection.target, selection.transientRetries) if err != nil { t.Fatal(err) } calls := 0 client := smokeInferenceClientFunc(func(_ context.Context, generationRequest domain.GenerateRequest) (*domain.GenerateResponse, error) { calls++ if generationRequest.TransientRetries != 0 || generationRequest.Generation != 1 || generationRequest.Target.BackendID != "alpha" { t.Fatalf("generation request selection/retries = %+v", generationRequest) } return &domain.GenerateResponse{Content: privateCase.expectedContent}, nil }) manager, err := capacity.NewManager(nil) if err != nil { t.Fatal(err) } runner.llm = generation.NewInjected(manager, client) runner.admitter = &smokeAdmissionSpy{} runner.defaultTransientRetries = 5 report, err := NewSmokeTester(runner, nil).Run(context.Background(), request) if err != nil { t.Fatal(err) } if calls != 1 || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone { t.Fatalf("calls=%d report=%+v", calls, report) } if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.EndpointReachable != nil { t.Fatalf("generation facts = %+v", report.Facts) } if kind == domain.SmokeTestTextInference { if report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || !*report.Facts.ExpectedTextMatched { t.Fatalf("text facts = %+v", report.Facts) } } else if report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested || report.Facts.ValidJSON == nil || !*report.Facts.ValidJSON || report.Facts.SchemaMatched == nil || !*report.Facts.SchemaMatched { t.Fatalf("structured facts = %+v", report.Facts) } if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].Timing.StartedAt.IsZero() || report.Attempts[0].Timing.CapacityWait == nil || report.Attempts[0].Timing.RequestDuration == nil || report.Timing.CapacityWait == nil || report.Timing.RequestDuration == nil || report.Timing.EvaluationDuration == nil { t.Fatalf("attempt/timing evidence = %+v", report) } if report.Usage.PromptTokens.Known || report.Attempts[0].Usage.PromptTokens.Known { t.Fatalf("unexpected usage evidence = %+v", report.Usage) } }) } } func TestSmokeTesterConcludesMissingRequiredCredentialWithoutDispatch(t *testing.T) { values := smokeSelectionConfig() profiles := values["profiles"].(map[string]any) profiles["required-profile"] = map[string]any{ "backend": "alpha", "model": "required-model", "api_key_required": true, } runner := smokeTesterRunner(t, values) admitter := &smokeAdmissionSpy{} runner.admitter = admitter report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: "required-profile", }) if err != nil { t.Fatal(err) } if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Reason != domain.SmokeTestReasonRequiredCredentialMissing || report.Failure != domain.ExecutionFailureInvalidRequest { t.Fatalf("missing-credential report = %+v", report) } if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || admitter.calls != 0 || !report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 { t.Fatalf("missing credential invented dispatch evidence: %+v admission calls=%d", report, admitter.calls) } } func TestSmokeTesterReportsStructuredContentValidationFindings(t *testing.T) { runner := smokeTesterRunner(t, smokeSelectionConfig()) request := domain.SmokeTestRequest{Kind: domain.SmokeTestStructuredOutput, ProfileID: "alpha-profile"} selection, err := runner.resolveSmokeTestTarget(context.Background(), request) if err != nil { t.Fatal(err) } privateCase, err := prepareSmokeInferenceCase(context.Background(), request.Kind, selection.target, selection.transientRetries) if err != nil { t.Fatal(err) } otherStatus := strings.Replace(privateCase.expectedContent, `"ok"`, `"other"`, 1) for _, tc := range []struct { name string content string wantValidJSON bool wantSchemaMatch *bool }{ {name: "invalid JSON", content: "not json", wantValidJSON: false}, {name: "schema mismatch", content: otherStatus, wantValidJSON: true, wantSchemaMatch: boolPointer(false)}, } { t.Run(tc.name, func(t *testing.T) { manager, err := capacity.NewManager(nil) if err != nil { t.Fatal(err) } runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { return &domain.GenerateResponse{Content: tc.content}, nil })) report, err := NewSmokeTester(runner, nil).Run(context.Background(), request) if err != nil { t.Fatal(err) } if !report.Completed || report.Failure != domain.ExecutionFailureValidation || report.Facts.ValidJSON == nil || *report.Facts.ValidJSON != tc.wantValidJSON || !sameBool(report.Facts.SchemaMatched, tc.wantSchemaMatch) { t.Fatalf("validation finding = %+v", report) } }) } } func TestSmokeTesterConcludesLogicalCapacityExhaustionWithoutDispatch(t *testing.T) { runner := smokeTesterRunner(t, smokeSelectionConfig()) manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{ "alpha": {ConcurrencyLimit: 1}, }) if err != nil { t.Fatal(err) } release, err := manager.Admit(context.Background(), "alpha") if err != nil { t.Fatal(err) } defer release() calls := 0 runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { calls++ return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil })) runner.admitter = manager report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", }) if err != nil { t.Fatal(err) } if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureCapacity { t.Fatalf("capacity report = %+v", report) } if calls != 0 || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || !report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 { t.Fatalf("capacity finding dispatched or invented facts: %+v calls=%d", report, calls) } } func TestSmokeTesterRetainsGenerationEvidenceWhenCallerCancelsBeforeEvaluation(t *testing.T) { runner := smokeTesterRunner(t, smokeSelectionConfig()) manager, err := capacity.NewManager(nil) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { cancel() return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil })) report, err := NewSmokeTester(runner, nil).Run(ctx, domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", }) if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled { t.Fatalf("partial report=%+v error=%v", report, err) } if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.TextReturned != nil || report.Facts.ExpectedTextMatched != nil || len(report.Attempts) != 1 { t.Fatalf("cancellation evidence = %+v", report) } } func TestSmokeTesterUsesZeroDefaultRetriesDespiteRunnerDefault(t *testing.T) { runner := smokeTesterRunner(t, smokeSelectionConfig()) manager, err := capacity.NewManager(nil) if err != nil { t.Fatal(err) } runner.llm = generation.NewBuiltIn(manager, reachabilityClient(t, func(*http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: ioNopCloser("retry must not be inherited"), }, nil })) runner.defaultTransientRetries = 5 report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile", }) if err != nil { t.Fatal(err) } if !report.Completed || report.Failure != domain.ExecutionFailureProviderStatus || len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable { t.Fatalf("retry-isolation report = %+v", report) } } func TestSmokeTesterAggregatesGenerationEvidenceWithoutReplacingLastDispatchFacts(t *testing.T) { status := http.StatusServiceUnavailable requested := true firstStart := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) secondStart := firstStart.Add(10 * time.Millisecond) usage := domain.ObservedTokenUsage{ PromptTokens: domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}, CompletionTokens: domain.ObservedTokenCount{Value: 9, Known: true, Complete: true}, } report := &domain.SmokeTestReport{Target: domain.SmokeTestTarget{Endpoint: "https://alpha.example/v1/chat/completions"}} tester := &SmokeTester{} tester.recordGenerationEvidence(report, generation.GenerationEvidence{ Attempts: []generation.AttemptEvidence{ { StartedAt: firstStart, Dispatched: true, ResponseReceived: true, HTTP: llm.HTTPExecutionEvidence{ DispatchInvoked: true, RequestObserved: true, Endpoint: report.Target.Endpoint, CredentialsSupplied: true, ResponseReceived: true, HTTPStatus: &status, }, CapacityWait: 2 * time.Millisecond, RequestDuration: 5 * time.Millisecond, Failure: domain.ExecutionFailureProviderStatus, Usage: usage, StructuredOutputRequested: &requested, }, { StartedAt: secondStart, CapacityWait: 7 * time.Millisecond, Failure: domain.ExecutionFailureCanceled, }, }, RetryDelays: []time.Duration{3 * time.Millisecond}, Usage: usage, }) if len(report.Attempts) != 2 || !report.Attempts[0].Dispatched || report.Attempts[1].Dispatched || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != status || !report.Attempts[0].Timing.StartedAt.Equal(firstStart) || !report.Attempts[1].Timing.StartedAt.Equal(secondStart) { t.Fatalf("attempt evidence = %+v", report.Attempts) } if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied || report.Facts.GenerationCompleted == nil || *report.Facts.GenerationCompleted || report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested { t.Fatalf("last dispatched facts were replaced: %+v", report.Facts) } if report.Timing.CapacityWait == nil || *report.Timing.CapacityWait != 9*time.Millisecond || report.Timing.RequestDuration == nil || *report.Timing.RequestDuration != 5*time.Millisecond || report.Timing.RetryDelay == nil || *report.Timing.RetryDelay != 3*time.Millisecond || report.Usage != usage { t.Fatalf("aggregated timing/usage = %+v usage=%+v", report.Timing, report.Usage) } } type smokeInferenceClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) func (f smokeInferenceClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) { return f(ctx, request) }