package usecase import ( "context" "errors" "fmt" "io" "net/http" "strings" "testing" "time" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/configuration" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/observation" ) func TestSmokeTesterAssemblesReachabilityReportWithoutGenerationAdmission(t *testing.T) { values := smokeSelectionConfig() alpha := values["backends"].(map[string]any)["alpha"].(map[string]any) alphaSmoke := alpha["smoke_tests"].(map[string]any) alphaReachability := alphaSmoke["reachability"].(map[string]any) alphaReachability["endpoint"] = "https://operator:private@example.invalid/health?check=1" runner := smokeTesterRunner(t, values) admitter := &smokeAdmissionSpy{} runner.admitter = admitter var observedURL string var authorization string body := &reachabilityProbeBody{} client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) { observedURL = request.URL.String() authorization = request.Header.Get("Authorization") return &http.Response{StatusCode: http.StatusUnauthorized, Header: make(http.Header), Body: body}, nil }) var observedEvents []domain.ExecutionEvent recorder, err := observation.New(func(event domain.ExecutionEvent) { observedEvents = append(observedEvents, event) }) if err != nil { t.Fatal(err) } recorder.Start() ctx := observation.WithRecorder(context.Background(), recorder) clock := smokeClock() report, err := newSmokeTesterWithClock(runner, client, clock).Run(ctx, domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "alpha", APIKey: "synthetic-secret", }) if err != nil { t.Fatal(err) } if !body.closed || observedURL != "https://example.invalid/health?check=1" || authorization != "" { t.Fatalf("request URL=%q authorization=%q body closed=%v", observedURL, authorization, body.closed) } if report.Kind != domain.SmokeTestReachability || !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone { t.Fatalf("report outcome = %+v", report) } if report.Target.Endpoint != "https://example.invalid/health?check=1" || report.Target.EffectiveSettings.Endpoint != "https://alpha.example/v1" || report.Target.EffectiveSettings.APIKey != "" { t.Fatalf("report target = %+v", report.Target) } if report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied { t.Fatalf("report facts = %+v", report.Facts) } if len(report.Attempts) != 1 || !report.Attempts[0].Dispatched || report.Attempts[0].InitialEndpoint != report.Target.Endpoint || report.Attempts[0].Endpoint != report.Target.Endpoint || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusUnauthorized { t.Fatalf("attempts = %+v", report.Attempts) } if report.Timing.StartedAt.IsZero() || report.Timing.Duration <= 0 || report.Timing.RequestDuration == nil || report.Attempts[0].Timing.RequestDuration == nil || report.Usage.PromptTokens.Known || report.Timing.CapacityWait != nil { t.Fatalf("report timing/usage = %+v", report) } if strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") || strings.Contains(fmt.Sprintf("%+v", report), "private") { t.Fatalf("report retained endpoint credentials: %+v", report) } if len(observedEvents) != 1 || observedEvents[0].Kind != domain.ExecutionStarted { t.Fatalf("diagnostic traffic entered inherited observer: %+v", observedEvents) } if admitter.calls != 0 { t.Fatalf("HTTP diagnostic acquired generation capacity %d times", admitter.calls) } } func TestSmokeTesterAssemblesAuthenticationRejectionReport(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) var authorization string client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) { authorization = request.Header.Get("Authorization") return &http.Response{ StatusCode: http.StatusUnauthorized, Header: make(http.Header), Body: ioNopCloser("provider body is not retained"), }, nil }) report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: "beta", APIKey: "synthetic-secret", }) if err != nil { t.Fatal(err) } if authorization != "Bearer synthetic-secret" { t.Fatalf("Authorization = %q", authorization) } if !report.Supported || report.Applicable == nil || !*report.Applicable || !report.Completed || report.Failure != domain.ExecutionFailureNone { t.Fatalf("report outcome = %+v", report) } if report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.CredentialsAccepted == nil || *report.Facts.CredentialsAccepted { t.Fatalf("report facts = %+v", report.Facts) } if len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusUnauthorized || report.Attempts[0].Endpoint != "https://beta.example/v1/key" { t.Fatalf("attempts = %+v", report.Attempts) } if report.Target.Endpoint != "https://beta.example/v1/key" || report.Target.EffectiveSettings.APIKey != "" || strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") || strings.Contains(fmt.Sprintf("%+v", report), "provider body") { t.Fatalf("report retained sensitive values: %+v", report) } } func TestSmokeTesterReportsUnsupportedAndInapplicableAuthentication(t *testing.T) { for _, tc := range []struct { name string backend string client bool configureNone bool supported bool applicable *bool reason domain.SmokeTestReason }{ { name: "unconfigured", backend: "epsilon", reason: domain.SmokeTestReasonProbeNotConfigured, applicable: nil, }, { name: "HTTP support absent", backend: "beta", reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable, applicable: nil, }, { name: "known inapplicable without HTTP support", backend: "epsilon", configureNone: true, supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable, }, } { t.Run(tc.name, func(t *testing.T) { values := smokeSelectionConfig() if tc.configureNone { backend := values["backends"].(map[string]any)[tc.backend].(map[string]any) backend["smoke_tests"].(map[string]any)["authentication"] = map[string]any{"kind": "none"} } runner := smokeTesterRunner(t, values) var client *llm.OpenAICompatibleClient if tc.client { client = reachabilityClient(t, func(*http.Request) (*http.Response, error) { t.Fatal("unsupported or inapplicable authentication dispatched") return nil, nil }) } report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: tc.backend, }) if err != nil { t.Fatal(err) } if report.Supported != tc.supported || report.Reason != tc.reason || !report.Completed || !sameBool(report.Applicable, tc.applicable) || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || report.Usage != noDispatchUsage() { t.Fatalf("report = %+v", report) } }) } } func TestSmokeTesterReportsReachabilityUnavailableWithoutHTTPClient(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) report, err := NewSmokeTester(runner, nil).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "alpha", }) if err != nil { t.Fatal(err) } if report.Supported || report.Applicable != nil || !report.Completed || report.Reason != domain.SmokeTestReasonHTTPDiagnosticsUnavailable || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || report.Usage != noDispatchUsage() { t.Fatalf("unsupported reachability report = %+v", report) } } func TestSmokeTesterSelectionAndCancellationDoNotDispatch(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) calls := 0 client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { calls++ return &http.Response{StatusCode: http.StatusNoContent, Header: make(http.Header), Body: &reachabilityProbeBody{}}, nil }) tester := NewSmokeTester(runner, client) if report, err := tester.Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "missing", }); report != nil || !errors.Is(err, ErrProfileLoad) { t.Fatalf("invalid selection report=%+v error=%v", report, err) } if calls != 0 { t.Fatalf("invalid selection dispatched %d requests", calls) } ctx, cancel := context.WithCancel(context.Background()) cancel() for _, kind := range []domain.SmokeTestKind{domain.SmokeTestReachability, domain.SmokeTestTextInference} { report, err := tester.Run(ctx, domain.SmokeTestRequest{ Kind: kind, BackendID: "alpha", }) if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled { t.Fatalf("canceled report=%+v error=%v", report, err) } if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || calls != 0 || report.Usage != noDispatchUsage() { t.Fatalf("cancellation dispatched, invented facts, or left usage unknown: %+v calls=%d", report, calls) } } } func TestSmokeTesterConfiguredTimeoutConcludesWithReport(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) timeout := 1 client := reachabilityClient(t, func(request *http.Request) (*http.Response, error) { <-request.Context().Done() return nil, request.Context().Err() }) report, err := NewSmokeTester(runner, client).Run(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "alpha", Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &timeout}, }) if err != nil { t.Fatal(err) } if !report.Completed || report.Failure != domain.ExecutionFailureDeadline || report.Facts.EndpointReachable == nil || *report.Facts.EndpointReachable || report.Facts.ResponseReceived == nil || *report.Facts.ResponseReceived || report.Facts.CredentialsSupplied == nil || *report.Facts.CredentialsSupplied { t.Fatalf("configured timeout report = %+v", report) } if len(report.Attempts) != 1 || report.Attempts[0].Failure != domain.ExecutionFailureDeadline || report.Attempts[0].HTTPStatus != nil || report.Timing.RequestDuration == nil { t.Fatalf("configured timeout attempt = %+v", report.Attempts) } } func TestSmokeTesterRetainsHTTPResponseEvidenceWhenCallerCancelsDuringBodyRead(t *testing.T) { for _, readErr := range []error{context.Canceled, io.ErrUnexpectedEOF} { t.Run(readErr.Error(), func(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: cancelingSmokeBody{cancel: cancel, err: readErr}, }, nil }) report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: "beta", APIKey: "synthetic-secret", }) if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled { t.Fatalf("partial report=%+v error=%v", report, err) } if report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || report.Facts.EndpointReachable == nil || !*report.Facts.EndpointReachable || report.Facts.CredentialsSupplied == nil || !*report.Facts.CredentialsSupplied || report.Facts.CredentialsAccepted != nil || len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus == nil || *report.Attempts[0].HTTPStatus != http.StatusOK { t.Fatalf("partial HTTP evidence = %+v", report) } if strings.Contains(fmt.Sprintf("%+v", report), "synthetic-secret") { t.Fatalf("partial report retained credential: %+v", report) } }) } } func TestSmokeTesterRecognizesCallerCancellationWithOpaqueHTTPError(t *testing.T) { for _, kind := range []domain.SmokeTestKind{domain.SmokeTestReachability, domain.SmokeTestAuthentication} { t.Run(string(kind), func(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { cancel() return nil, io.ErrUnexpectedEOF }) report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{ Kind: kind, BackendID: "beta", }) if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled { t.Fatalf("partial report=%+v error=%v", report, err) } if len(report.Attempts) != 1 || report.Attempts[0].HTTPStatus != nil || report.Facts.ResponseReceived == nil || *report.Facts.ResponseReceived || report.Facts.CredentialsAccepted != nil { t.Fatalf("partial HTTP evidence = %+v", report) } }) } } func TestSmokeTesterKeepsCompletedAuthenticationWhenCloseCancelsCaller(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: cancelOnCloseSmokeBody{ ReadCloser: io.NopCloser(strings.NewReader(`{"data":{"label":"member","is_free_tier":false,"usage":1}}`)), cancel: cancel, }, }, nil }) report, err := NewSmokeTester(runner, client).Run(ctx, domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: "beta", }) if !errors.Is(ctx.Err(), context.Canceled) || err != nil || report == nil || !report.Completed || report.Failure != domain.ExecutionFailureNone || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted { t.Fatalf("completed report=%+v error=%v context error=%v", report, err, ctx.Err()) } } func smokeTesterRunner(t *testing.T, values map[string]any) *Runner { t.Helper() loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("smoke-test", values)}, nil) if err != nil { t.Fatal(err) } return NewRunner(nil, loaded, nil, nil, nil, nil, nil) } func smokeClock() func() time.Time { current := time.Date(2026, time.October, 13, 12, 0, 0, 0, time.UTC) return func() time.Time { current = current.Add(time.Millisecond) return current } } type smokeAdmissionSpy struct{ calls int } func (a *smokeAdmissionSpy) Admit(context.Context, string) (func(), error) { a.calls++ return func() {}, nil } type cancelingSmokeBody struct { cancel context.CancelFunc err error } func (b cancelingSmokeBody) Read(buffer []byte) (int, error) { b.cancel() return copy(buffer, []byte(`{"data":{"label":"member","is_free_tier":false,"usage":1}}`)), b.err } func (cancelingSmokeBody) Close() error { return nil } type cancelOnCloseSmokeBody struct { io.ReadCloser cancel context.CancelFunc } func (b cancelOnCloseSmokeBody) Close() error { b.cancel() return b.ReadCloser.Close() }