396 lines
16 KiB
Go
396 lines
16 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
|
)
|
|
|
|
func TestSmokeTesterAssemblesReachabilityReportWithoutGenerationAdmission(t *testing.T) {
|
|
values := smokeSelectionConfig()
|
|
alpha := values["backends"].(map[string]any)["alpha"].(map[string]any)
|
|
alphaSmoke := alpha["smoke_tests"].(map[string]any)
|
|
alphaReachability := alphaSmoke["reachability"].(map[string]any)
|
|
alphaReachability["endpoint"] = "https://operator:private@example.invalid/health?check=1"
|
|
runner := smokeTesterRunner(t, values)
|
|
admitter := &smokeAdmissionSpy{}
|
|
runner.admitter = admitter
|
|
|
|
var observedURL string
|
|
var authorization string
|
|
body := &reachabilityProbeBody{}
|
|
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
|
|
observedURL = request.URL.String()
|
|
authorization = request.Header.Get("Authorization")
|
|
return &http.Response{StatusCode: http.StatusUnauthorized, Header: make(http.Header), Body: body}, nil
|
|
})
|
|
var observedEvents []domain.ExecutionEvent
|
|
recorder, err := observation.New(func(event domain.ExecutionEvent) { observedEvents = append(observedEvents, event) })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recorder.Start()
|
|
ctx := observation.WithRecorder(context.Background(), recorder)
|
|
|
|
clock := smokeClock()
|
|
report, err := newSmokeTesterWithClock(runner, client, clock).Run(ctx, domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability,
|
|
BackendID: "alpha",
|
|
APIKey: "synthetic-secret",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !body.closed || observedURL != "https://example.invalid/health?check=1" || authorization != "" {
|
|
t.Fatalf("request URL=%q authorization=%q body closed=%v", observedURL, authorization, body.closed)
|
|
}
|
|
if report.Kind != domain.SmokeTestReachability || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone {
|
|
t.Fatalf("report outcome = %+v", report)
|
|
}
|
|
if report.Target.Endpoint != "https://example.invalid/health?check=1" ||
|
|
report.Target.EffectiveSettings.Endpoint != "https://alpha.example/v1" ||
|
|
report.Target.EffectiveSettings.APIKey != "" {
|
|
t.Fatalf("report target = %+v", report.Target)
|
|
}
|
|
if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
|
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied {
|
|
t.Fatalf("report facts = %+v", report.Facts)
|
|
}
|
|
if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched ||
|
|
report.Attempts[0].InitialEndpoint != report.Target.Endpoint ||
|
|
report.Attempts[0].Endpoint != report.Target.Endpoint ||
|
|
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusUnauthorized {
|
|
t.Fatalf("attempts = %+v", report.Attempts)
|
|
}
|
|
if report.Timing.StartedAt.IsZero() || report.Timing.Duration <= 0 || report.Timing.RequestDuration == nil ||
|
|
report.Attempts[0].Timing.RequestDuration == nil || report.Usage.PromptTokens.Known || report.Timing.CapacityWait != nil {
|
|
t.Fatalf("report timing/usage = %+v", report)
|
|
}
|
|
if strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") || strings.Contains(fmt.Sprintf("%+v", report), "private") {
|
|
t.Fatalf("report retained endpoint credentials: %+v", report)
|
|
}
|
|
if len(observedEvents) != 1 || observedEvents[0].Kind != domain.ExecutionStarted {
|
|
t.Fatalf("diagnostic traffic entered inherited observer: %+v", observedEvents)
|
|
}
|
|
if admitter.calls != 0 {
|
|
t.Fatalf("HTTP diagnostic acquired generation capacity %d times", admitter.calls)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterAssemblesAuthenticationRejectionReport(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
var authorization string
|
|
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
|
|
authorization = request.Header.Get("Authorization")
|
|
return &http.Response{
|
|
StatusCode: http.StatusUnauthorized,
|
|
Header: make(http.Header),
|
|
Body: ioNopCloser("provider body is not retained"),
|
|
}, nil
|
|
})
|
|
|
|
report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestAuthentication,
|
|
BackendID: "beta",
|
|
APIKey: "synthetic-secret",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if authorization != "Bearer synthetic-secret" {
|
|
t.Fatalf("Authorization = %q", authorization)
|
|
}
|
|
if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone {
|
|
t.Fatalf("report outcome = %+v", report)
|
|
}
|
|
if report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied ||
|
|
report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
|
report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.CredentialsAccepted == nil || *report.Facts.CredentialsAccepted {
|
|
t.Fatalf("report facts = %+v", report.Facts)
|
|
}
|
|
if len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusUnauthorized ||
|
|
report.Attempts[0].Endpoint != "https://beta.example/v1/key" {
|
|
t.Fatalf("attempts = %+v", report.Attempts)
|
|
}
|
|
if report.Target.Endpoint != "https://beta.example/v1/key" || report.Target.EffectiveSettings.APIKey != "" ||
|
|
strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") ||
|
|
strings.Contains(fmt.Sprintf("%+v", report), "provider body") {
|
|
t.Fatalf("report retained sensitive values: %+v", report)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterReportsUnsupportedAndInapplicableAuthentication(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
backend string
|
|
client bool
|
|
configureNone bool
|
|
supported bool
|
|
applicable *bool
|
|
reason domain.SmokeTestReason
|
|
}{
|
|
{
|
|
name: "unconfigured",
|
|
backend: "epsilon",
|
|
reason: domain.SmokeTestReasonProbeNotConfigured,
|
|
applicable: nil,
|
|
},
|
|
{
|
|
name: "HTTP support absent",
|
|
backend: "beta",
|
|
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
|
applicable: nil,
|
|
},
|
|
{
|
|
name: "known inapplicable without HTTP support",
|
|
backend: "epsilon",
|
|
configureNone: true,
|
|
supported: true,
|
|
applicable: boolPointer(false),
|
|
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
values := smokeSelectionConfig()
|
|
if tc.configureNone {
|
|
backend := values["backends"].(map[string]any)[tc.backend].(map[string]any)
|
|
backend["smoke_tests"].(map[string]any)["authentication"] = map[string]any{"kind": "none"}
|
|
}
|
|
runner := smokeTesterRunner(t, values)
|
|
var client *llm.OpenAICompatibleClient
|
|
if tc.client {
|
|
client = reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
t.Fatal("unsupported or inapplicable authentication dispatched")
|
|
return nil, nil
|
|
})
|
|
}
|
|
report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestAuthentication, BackendID: tc.backend,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.Supported != tc.supported || report.Reason != tc.reason || !report.Completed ||
|
|
!sameBool(report.Applicable, tc.applicable) || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) ||
|
|
report.Usage != noDispatchUsage() {
|
|
t.Fatalf("report = %+v", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterReportsReachabilityUnavailableWithoutHTTPClient(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, BackendID: "alpha",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.Supported || report.Applicable != nil || !report.Completed ||
|
|
report.Reason != domain.SmokeTestReasonHTTPDiagnosticsUnavailable ||
|
|
len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || report.Usage != noDispatchUsage() {
|
|
t.Fatalf("unsupported reachability report = %+v", report)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterSelectionAndCancellationDoNotDispatch(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
calls := 0
|
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
calls++
|
|
return &http.Response{StatusCode: http.StatusNoContent, Header: make(http.Header), Body: &reachabilityProbeBody{}}, nil
|
|
})
|
|
tester := NewSmokeTester(runner, client)
|
|
|
|
if report, err := tester.Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, BackendID: "missing",
|
|
}); report != nil || !errors.Is(err, ErrProfileLoad) {
|
|
t.Fatalf("invalid selection report=%+v error=%v", report, err)
|
|
}
|
|
if calls != 0 {
|
|
t.Fatalf("invalid selection dispatched %d requests", calls)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
for _, kind := range []domain.SmokeTestKind{domain.SmokeTestReachability, domain.SmokeTestTextInference} {
|
|
report, err := tester.Run(ctx, domain.SmokeTestRequest{
|
|
Kind: kind, BackendID: "alpha",
|
|
})
|
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
|
|
t.Fatalf("canceled report=%+v error=%v", report, err)
|
|
}
|
|
if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || calls != 0 || report.Usage != noDispatchUsage() {
|
|
t.Fatalf("cancellation dispatched, invented facts, or left usage unknown: %+v calls=%d", report, calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterConfiguredTimeoutConcludesWithReport(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
timeout := 1
|
|
client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) {
|
|
<-request.Context().Done()
|
|
return nil, request.Context().Err()
|
|
})
|
|
report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestReachability, BackendID: "alpha",
|
|
Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &timeout},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.Completed || report.Failure != domain.ExecutionFailureDeadline ||
|
|
report.Facts.EndpointReachable == nil || *report.Facts.EndpointReachable ||
|
|
report.Facts.ResponseReceived == nil || *report.Facts.ResponseReceived ||
|
|
report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied {
|
|
t.Fatalf("configured timeout report = %+v", report)
|
|
}
|
|
if len(report.Attempts) != 1 || report.Attempts[0].Failure != domain.ExecutionFailureDeadline ||
|
|
report.Attempts[0].HTTPStatus != nil || report.Timing.RequestDuration == nil {
|
|
t.Fatalf("configured timeout attempt = %+v", report.Attempts)
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterRetainsHTTPResponseEvidenceWhenCallerCancelsDuringBodyRead(t *testing.T) {
|
|
for _, readErr := range []error{context.Canceled, io.ErrUnexpectedEOF} {
|
|
t.Run(readErr.Error(), func(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Header: make(http.Header),
|
|
Body: cancelingSmokeBody{cancel: cancel, err: readErr},
|
|
}, nil
|
|
})
|
|
|
|
report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestAuthentication, BackendID: "beta", APIKey: "synthetic-secret",
|
|
})
|
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
|
|
t.Fatalf("partial report=%+v error=%v", report, err)
|
|
}
|
|
if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived ||
|
|
report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable ||
|
|
report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied ||
|
|
report.Facts.CredentialsAccepted != nil || len(report.Attempts) != 1 ||
|
|
report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusOK {
|
|
t.Fatalf("partial HTTP evidence = %+v", report)
|
|
}
|
|
if strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") {
|
|
t.Fatalf("partial report retained credential: %+v", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterRecognizesCallerCancellationWithOpaqueHTTPError(t *testing.T) {
|
|
for _, kind := range []domain.SmokeTestKind{domain.SmokeTestReachability, domain.SmokeTestAuthentication} {
|
|
t.Run(string(kind), func(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
cancel()
|
|
return nil, io.ErrUnexpectedEOF
|
|
})
|
|
report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{
|
|
Kind: kind, BackendID: "beta",
|
|
})
|
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
|
|
t.Fatalf("partial report=%+v error=%v", report, err)
|
|
}
|
|
if len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus != nil ||
|
|
report.Facts.ResponseReceived == nil || *report.Facts.ResponseReceived ||
|
|
report.Facts.CredentialsAccepted != nil {
|
|
t.Fatalf("partial HTTP evidence = %+v", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeTesterKeepsCompletedAuthenticationWhenCloseCancelsCaller(t *testing.T) {
|
|
runner, _ := newSmokeSelectionRunner(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Header: make(http.Header),
|
|
Body: cancelOnCloseSmokeBody{
|
|
ReadCloser: io.NopCloser(strings.NewReader(`{"data":{"label":"member","is_free_tier":false,"usage":1}}`)),
|
|
cancel: cancel,
|
|
},
|
|
}, nil
|
|
})
|
|
report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{
|
|
Kind: domain.SmokeTestAuthentication, BackendID: "beta",
|
|
})
|
|
if !errors.Is(ctx.Err(), context.Canceled) || err != nil || report == nil || !report.Completed ||
|
|
report.Failure != domain.ExecutionFailureNone || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted {
|
|
t.Fatalf("completed report=%+v error=%v context error=%v", report, err, ctx.Err())
|
|
}
|
|
}
|
|
|
|
func smokeTesterRunner(t *testing.T, values map[string]any) *Runner {
|
|
t.Helper()
|
|
loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("smoke-test", values)}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return NewRunner(nil, loaded, nil, nil, nil, nil, nil)
|
|
}
|
|
|
|
func smokeClock() func() time.Time {
|
|
current := time.Date(2026, time.October, 13, 12, 0, 0, 0, time.UTC)
|
|
return func() time.Time {
|
|
current = current.Add(time.Millisecond)
|
|
return current
|
|
}
|
|
}
|
|
|
|
type smokeAdmissionSpy struct{ calls int }
|
|
|
|
func (a *smokeAdmissionSpy) Admit(context.Context, string) (func(), error) {
|
|
a.calls++
|
|
return func() {}, nil
|
|
}
|
|
|
|
type cancelingSmokeBody struct {
|
|
cancel context.CancelFunc
|
|
err error
|
|
}
|
|
|
|
func (b cancelingSmokeBody) Read(buffer []byte) (int, error) {
|
|
b.cancel()
|
|
return copy(buffer, []byte(`{"data":{"label":"member","is_free_tier":false,"usage":1}}`)), b.err
|
|
}
|
|
|
|
func (cancelingSmokeBody) Close() error { return nil }
|
|
|
|
type cancelOnCloseSmokeBody struct {
|
|
io.ReadCloser
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
func (b cancelOnCloseSmokeBody) Close() error {
|
|
b.cancel()
|
|
return b.ReadCloser.Close()
|
|
}
|