289 lines
12 KiB
Go
289 lines
12 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
|
)
|
|
|
|
func TestSmokeTesterRunsPrivateInferenceCasesThroughInjectedCoordinator(t *testing.T) {
|
|
for _, kind := range []domain.SmokeTestKind{domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput} {
|
|
t.Run(string(kind), func(t *testing.T) {
|
|
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
|
request := domain.SmokeTestRequest{Kind: kind, ProfileID: "alpha-profile"}
|
|
selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
privateCase, err := prepareSmokeInferenceCase(context.Background(), kind, selection.target, selection.transientRetries)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := 0
|
|
client := smokeInferenceClientFunc(func(_ context.Context, generationRequest domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
calls++
|
|
if generationRequest.TransientRetries != 0 || generationRequest.Generation != 1 || generationRequest.Target.BackendID != "alpha" {
|
|
t.Fatalf("generation request selection/retries = %+v", generationRequest)
|
|
}
|
|
return &domain.GenerateResponse{Content: privateCase.expectedContent}, nil
|
|
})
|
|
manager, err := capacity.NewManager(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.llm = generation.NewInjected(manager, client)
|
|
runner.admitter = &smokeAdmissionSpy{}
|
|
runner.defaultTransientRetries = 5
|
|
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls != 1 || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone {
|
|
t.Fatalf("calls=%d report=%+v", calls, report)
|
|
}
|
|
if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.EndpointReachable != nil {
|
|
t.Fatalf("generation facts = %+v", report.Facts)
|
|
}
|
|
if kind == domain.SmokeTestTextInference {
|
|
if report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || !*report.Facts.ExpectedTextMatched {
|
|
t.Fatalf("text facts = %+v", report.Facts)
|
|
}
|
|
} else if report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested ||
|
|
report.Facts.ValidJSON == nil || !*report.Facts.ValidJSON || report.Facts.SchemaMatched == nil || !*report.Facts.SchemaMatched {
|
|
t.Fatalf("structured facts = %+v", report.Facts)
|
|
}
|
|
if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].Timing.StartedAt.IsZero() ||
|
|
report.Attempts[0].Timing.CapacityWait == nil || report.Attempts[0].Timing.RequestDuration == nil ||
|
|
report.Timing.CapacityWait == nil || report.Timing.RequestDuration == nil || report.Timing.EvaluationDuration == nil {
|
|
t.Fatalf("attempt/timing evidence = %+v", report)
|
|
}
|
|
if report.Usage.PromptTokens.Known || report.Attempts[0].Usage.PromptTokens.Known {
|
|
t.Fatalf("unexpected usage evidence = %+v", report.Usage)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterConcludesMissingRequiredCredentialWithoutDispatch(t *testing.T) {
|
|
values := smokeSelectionConfig()
|
|
profiles := values["profiles"].(map[string]any)
|
|
profiles["required-profile"] = map[string]any{
|
|
"backend": "alpha", "model": "required-model", "api_key_required": true,
|
|
}
|
|
runner := smokeTesterRunner(t, values)
|
|
admitter := &smokeAdmissionSpy{}
|
|
runner.admitter = admitter
|
|
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: "required-profile",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed ||
|
|
report.Reason != domain.SmokeTestReasonRequiredCredentialMissing || report.Failure != domain.ExecutionFailureInvalidRequest {
|
|
t.Fatalf("missing-credential report = %+v", report)
|
|
}
|
|
if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || admitter.calls != 0 ||
|
|
!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
|
|
t.Fatalf("missing credential invented dispatch evidence: %+v admission calls=%d", report, admitter.calls)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterReportsStructuredContentValidationFindings(t *testing.T) {
|
|
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
|
request := domain.SmokeTestRequest{Kind: domain.SmokeTestStructuredOutput, ProfileID: "alpha-profile"}
|
|
selection, err := runner.resolveSmokeTestTarget(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
privateCase, err := prepareSmokeInferenceCase(context.Background(), request.Kind, selection.target, selection.transientRetries)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
otherStatus := strings.Replace(privateCase.expectedContent, `"ok"`, `"other"`, 1)
|
|
for _, tc := range []struct {
|
|
name string
|
|
content string
|
|
wantValidJSON bool
|
|
wantSchemaMatch *bool
|
|
}{
|
|
{name: "invalid JSON", content: "not json", wantValidJSON: false},
|
|
{name: "schema mismatch", content: otherStatus, wantValidJSON: true, wantSchemaMatch: boolPointer(false)},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
manager, err := capacity.NewManager(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
return &domain.GenerateResponse{Content: tc.content}, nil
|
|
}))
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.Completed || report.Failure != domain.ExecutionFailureValidation ||
|
|
report.Facts.ValidJSON == nil || *report.Facts.ValidJSON != tc.wantValidJSON ||
|
|
!sameBool(report.Facts.SchemaMatched, tc.wantSchemaMatch) {
|
|
t.Fatalf("validation finding = %+v", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterConcludesLogicalCapacityExhaustionWithoutDispatch(t *testing.T) {
|
|
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
|
manager, err := capacity.NewManager(map[string]domain.BackendCapacityPolicy{
|
|
"alpha": {ConcurrencyLimit: 1},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
release, err := manager.Admit(context.Background(), "alpha")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer release()
|
|
calls := 0
|
|
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
calls++
|
|
return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
|
|
}))
|
|
runner.admitter = manager
|
|
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureCapacity {
|
|
t.Fatalf("capacity report = %+v", report)
|
|
}
|
|
if calls != 0 || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) ||
|
|
!report.Usage.PromptTokens.Known || !report.Usage.PromptTokens.Complete || report.Usage.PromptTokens.Value != 0 {
|
|
t.Fatalf("capacity finding dispatched or invented facts: %+v calls=%d", report, calls)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterRetainsGenerationEvidenceWhenCallerCancelsBeforeEvaluation(t *testing.T) {
|
|
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
|
manager, err := capacity.NewManager(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
runner.llm = generation.NewInjected(manager, smokeInferenceClientFunc(func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
cancel()
|
|
return &domain.GenerateResponse{Content: smokeTextExpectedContent}, nil
|
|
}))
|
|
|
|
report, err := NewSmokeTester(runner, nil).Run(ctx, domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
|
})
|
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
|
|
t.Fatalf("partial report=%+v error=%v", report, err)
|
|
}
|
|
if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
|
|
report.Facts.TextReturned != nil || report.Facts.ExpectedTextMatched != nil || len(report.Attempts) != 1 {
|
|
t.Fatalf("cancellation evidence = %+v", report)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterUsesZeroDefaultRetriesDespiteRunnerDefault(t *testing.T) {
|
|
runner := smokeTesterRunner(t, smokeSelectionConfig())
|
|
manager, err := capacity.NewManager(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner.llm = generation.NewBuiltIn(manager, reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
return &http.Response{
|
|
StatusCode: http.StatusServiceUnavailable,
|
|
Header: make(http.Header),
|
|
Body: ioNopCloser("retry must not be inherited"),
|
|
}, nil
|
|
}))
|
|
runner.defaultTransientRetries = 5
|
|
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.Completed || report.Failure != domain.ExecutionFailureProviderStatus || len(report.Attempts) != 1 ||
|
|
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusServiceUnavailable {
|
|
t.Fatalf("retry-isolation report = %+v", report)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterAggregatesGenerationEvidenceWithoutReplacingLastDispatchFacts(t *testing.T) {
|
|
status := http.StatusServiceUnavailable
|
|
requested := true
|
|
firstStart := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
|
|
secondStart := firstStart.Add(10 * time.Millisecond)
|
|
usage := domain.ObservedTokenUsage{
|
|
PromptTokens: domain.ObservedTokenCount{Value: 4, Known: true, Complete: false},
|
|
CompletionTokens: domain.ObservedTokenCount{Value: 9, Known: true, Complete: true},
|
|
}
|
|
report := &domain.SmokeTestReport{Target: domain.SmokeTestTarget{Endpoint: "https://alpha.example/v1/chat/completions"}}
|
|
tester := &SmokeTester{}
|
|
tester.recordGenerationEvidence(report, generation.GenerationEvidence{
|
|
Attempts: []generation.AttemptEvidence{
|
|
{
|
|
StartedAt: firstStart, Dispatched: true, ResponseReceived: true,
|
|
HTTP: llm.HTTPExecutionEvidence{
|
|
DispatchInvoked: true, RequestObserved: true, Endpoint: report.Target.Endpoint,
|
|
CredentialsSupplied: true, ResponseReceived: true, HTTPStatus: &status,
|
|
},
|
|
CapacityWait: 2 * time.Millisecond, RequestDuration: 5 * time.Millisecond,
|
|
Failure: domain.ExecutionFailureProviderStatus, Usage: usage,
|
|
StructuredOutputRequested: &requested,
|
|
},
|
|
{
|
|
StartedAt: secondStart, CapacityWait: 7 * time.Millisecond,
|
|
Failure: domain.ExecutionFailureCanceled,
|
|
},
|
|
},
|
|
RetryDelays: []time.Duration{3 * time.Millisecond},
|
|
Usage: usage,
|
|
})
|
|
|
|
if len(report.Attempts) != 2 || !report.Attempts[0].Dispatched || report.Attempts[1].Dispatched ||
|
|
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != status ||
|
|
!report.Attempts[0].Timing.StartedAt.Equal(firstStart) || !report.Attempts[1].Timing.StartedAt.Equal(secondStart) {
|
|
t.Fatalf("attempt evidence = %+v", report.Attempts)
|
|
}
|
|
if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
|
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied ||
|
|
report.Facts.GenerationCompleted == nil || *report.Facts.GenerationCompleted ||
|
|
report.Facts.StructuredOutputRequested == nil || !*report.Facts.StructuredOutputRequested {
|
|
t.Fatalf("last dispatched facts were replaced: %+v", report.Facts)
|
|
}
|
|
if report.Timing.CapacityWait == nil || *report.Timing.CapacityWait != 9*time.Millisecond ||
|
|
report.Timing.RequestDuration == nil || *report.Timing.RequestDuration != 5*time.Millisecond ||
|
|
report.Timing.RetryDelay == nil || *report.Timing.RetryDelay != 3*time.Millisecond ||
|
|
report.Usage != usage {
|
|
t.Fatalf("aggregated timing/usage = %+v usage=%+v", report.Timing, report.Usage)
|
|
}
|
|
}
|
|
|
|
type smokeInferenceClientFunc func(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error)
|
|
|
|
func (f smokeInferenceClientFunc) Generate(ctx context.Context, request domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
return f(ctx, request)
|
|
}
|