package usecase import ( "context" "errors" "fmt" "time" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/generation" "gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/observation" ) // SmokeTester coordinates internal diagnostic operations over captured // selection collaborators, shared generation, and optional built-in HTTP. type SmokeTester struct { runner *Runner httpClient smokeHTTPGetter now func() time.Time } // NewSmokeTester creates an internal diagnostic coordinator. A nil HTTP client // leaves HTTP operations unsupported rather than falling back to another client. func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester { var getter smokeHTTPGetter if httpClient != nil { getter = httpClient } return &SmokeTester{runner: runner, httpClient: getter, now: time.Now} } func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester { var getter smokeHTTPGetter if httpClient != nil { getter = httpClient } return &SmokeTester{runner: runner, httpClient: getter, now: now} } // Run selects and executes one diagnostic, retaining only factual results. func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) { if t == nil || t.runner == nil { return nil, fmt.Errorf("%w: smoke tester is not configured", ErrInvalidRequest) } ctx = observation.WithoutExecutionState(ctx) now := t.now if now == nil { now = time.Now } startedAt := now().UTC() report := &domain.SmokeTestReport{ Kind: request.Kind, Timing: domain.SmokeTestTiming{ StartedAt: startedAt, }, } finish := func(err error) (*domain.SmokeTestReport, error) { report.Timing.Duration = nonnegativeDuration(now().UTC().Sub(startedAt)) for _, attempt := range report.Attempts { if attempt.Dispatched { return report, err } } report.Usage = noDispatchUsage() return report, err } selection, err := t.runner.resolveSmokeTestTarget(ctx, request) if err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } return nil, err } report.Target, err = smokeReportTarget(selection.target) if err != nil { return nil, err } if isInferenceSmokeTest(request.Kind) { // The configured base is not evidence of an HTTP request destination. // Built-in generation supplies its composed URL after preparation. report.Target.Endpoint = "" report.Supported = true report.Applicable = boolPointer(true) } else if err := t.setSupport(report, selection); err != nil { return nil, err } if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } if !report.Supported || (report.Applicable != nil && !*report.Applicable) { report.Completed = true return finish(nil) } switch request.Kind { case domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput: return t.runInference(ctx, request, selection, report, now, finish) case domain.SmokeTestReachability: requestStartedAt := now().UTC() outcome, err := (reachabilityProbe{client: t.httpClient}).run( ctx, selection.target.Endpoint, selection.target.EffectiveSettings.TimeoutSeconds, ) requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt)) if err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } return nil, err } report.Supported = outcome.supported report.Reason = outcome.reason report.Failure = outcome.failure report.Facts = httpExecutionFacts(outcome.evidence) if outcome.reachable != nil { reachable := *outcome.reachable report.Facts.EndpointReachable = &reachable } t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil) return t.finishHTTP(ctx, report, finish) case domain.SmokeTestAuthentication: requestStartedAt := now().UTC() outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target) requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt)) if err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } return nil, err } report.Supported = outcome.supported report.Applicable = cloneOptionalBool(outcome.applicable) report.Reason = outcome.reason report.Failure = outcome.failure report.Facts = httpExecutionFacts(outcome.evidence) report.Facts.CredentialsAccepted = cloneOptionalBool(outcome.credentialsAccepted) t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, outcome.credentialsAccepted) return t.finishHTTP(ctx, report, finish) default: return nil, fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, request.Kind) } } type smokeEvidenceGenerator interface { GenerateWithEvidence(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error, generation.GenerationEvidence) } func (t *SmokeTester) runInference( ctx context.Context, request domain.SmokeTestRequest, selection *smokeTestSelection, report *domain.SmokeTestReport, now func() time.Time, finish func(error) (*domain.SmokeTestReport, error), ) (*domain.SmokeTestReport, error) { settings := selection.target.EffectiveSettings if err := validateAPIKey(settings.APIKeyEnv, settings.APIKey, settings.APIKeyRequired); err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) { report.Reason = domain.SmokeTestReasonRequiredCredentialMissing report.Failure = domain.ExecutionFailureInvalidRequest report.Completed = true return finish(nil) } return nil, err } inferenceCase, err := prepareSmokeInferenceCase(ctx, request.Kind, selection.target, selection.transientRetries) if err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } return nil, err } generator, ok := t.runner.llm.(smokeEvidenceGenerator) if !ok { return nil, fmt.Errorf("smoke-test generation evidence is unavailable") } release, err := t.runner.admitRun(ctx, selection.target.BackendID) if err != nil { if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } var capacityErr *CapacityError if errors.As(err, &capacityErr) { report.Failure = domain.ExecutionFailureCapacity report.Completed = true return finish(nil) } return nil, fmt.Errorf("admit smoke-test generation: %w", err) } defer release() response, generationErr, evidence := generator.GenerateWithEvidence(ctx, inferenceCase.request) t.recordGenerationEvidence(report, evidence) if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } if generationErr != nil { report.Failure = generation.ClassifyFailure(generationErr) report.Completed = true return finish(nil) } if response == nil { report.Failure = generation.ClassifyFailure(llm.ErrMalformedResponse) report.Completed = true return finish(nil) } evaluationStartedAt := now().UTC() evaluatedFacts, evaluationErr := inferenceCase.evaluate(ctx, response) evaluationDuration := nonnegativeDuration(now().UTC().Sub(evaluationStartedAt)) report.Timing.EvaluationDuration = &evaluationDuration mergeEvaluatedSmokeFacts(&report.Facts, evaluatedFacts) if contextErr := ctx.Err(); contextErr != nil { report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } if evaluationErr != nil { return nil, fmt.Errorf("evaluate smoke-test response: %w", evaluationErr) } report.Completed = true report.Failure = domain.ExecutionFailureNone if request.Kind == domain.SmokeTestStructuredOutput && ((report.Facts.ValidJSON != nil && !*report.Facts.ValidJSON) || (report.Facts.SchemaMatched != nil && !*report.Facts.SchemaMatched)) { report.Failure = domain.ExecutionFailureValidation } return finish(nil) } func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) { report.Target.Endpoint = evidence.InitialEndpoint report.Usage = evidence.Usage var capacityWait, requestDuration time.Duration for _, attemptEvidence := range evidence.Attempts { capacityWait += attemptEvidence.CapacityWait attempt := domain.SmokeTestAttempt{ Dispatched: attemptEvidence.Dispatched, InitialEndpoint: evidence.InitialEndpoint, Endpoint: attemptEvidence.HTTP.Endpoint, Facts: generationAttemptFacts(attemptEvidence), Usage: attemptEvidence.Usage, Failure: attemptEvidence.Failure, Timing: domain.SmokeTestTiming{ StartedAt: attemptEvidence.StartedAt, Duration: nonnegativeDuration(attemptEvidence.CapacityWait + attemptEvidence.RequestDuration), CapacityWait: durationPointer(attemptEvidence.CapacityWait), }, } if attemptEvidence.Dispatched { requestDuration += attemptEvidence.RequestDuration attempt.Timing.RequestDuration = durationPointer(attemptEvidence.RequestDuration) } if attemptEvidence.HTTP.HTTPStatus != nil { status := *attemptEvidence.HTTP.HTTPStatus attempt.HTTPStatus = &status } report.Attempts = append(report.Attempts, attempt) if attemptEvidence.Dispatched { report.Facts = generationAttemptFacts(attemptEvidence) } } if len(evidence.Attempts) > 0 { report.Timing.CapacityWait = durationPointer(capacityWait) } if requestDurationKnown(evidence.Attempts) { report.Timing.RequestDuration = durationPointer(requestDuration) } if len(evidence.RetryDelays) > 0 { var retryDelay time.Duration for _, delay := range evidence.RetryDelays { retryDelay += delay } report.Timing.RetryDelay = durationPointer(retryDelay) } } func generationAttemptFacts(evidence generation.AttemptEvidence) domain.SmokeTestFacts { facts := httpExecutionFacts(evidence.HTTP) if evidence.Dispatched { responseReceived := evidence.ResponseReceived generationCompleted := evidence.GenerationCompleted facts.ResponseReceived = &responseReceived facts.GenerationCompleted = &generationCompleted } facts.StructuredOutputRequested = cloneOptionalBool(evidence.StructuredOutputRequested) return facts } func mergeEvaluatedSmokeFacts(target *domain.SmokeTestFacts, evaluated domain.SmokeTestFacts) { if evaluated.TextReturned != nil { target.TextReturned = cloneOptionalBool(evaluated.TextReturned) } if evaluated.ExpectedTextMatched != nil { target.ExpectedTextMatched = cloneOptionalBool(evaluated.ExpectedTextMatched) } if evaluated.ValidJSON != nil { target.ValidJSON = cloneOptionalBool(evaluated.ValidJSON) } if evaluated.SchemaMatched != nil { target.SchemaMatched = cloneOptionalBool(evaluated.SchemaMatched) } } func noDispatchUsage() domain.ObservedTokenUsage { return (observation.UsageAccumulator{}).Observed() } func durationPointer(value time.Duration) *time.Duration { cloned := value return &cloned } func requestDurationKnown(attempts []generation.AttemptEvidence) bool { for _, attempt := range attempts { if attempt.Dispatched { return true } } return false } func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error { switch report.Kind { case domain.SmokeTestReachability: if t.httpClient == nil { report.Reason = domain.SmokeTestReasonHTTPDiagnosticsUnavailable return nil } report.Supported = true report.Applicable = boolPointer(true) return nil case domain.SmokeTestAuthentication: outcome, err := (authenticationProbe{client: t.httpClient}).support(selection.backend) if err != nil { return err } report.Supported = outcome.supported report.Applicable = cloneOptionalBool(outcome.applicable) report.Reason = outcome.reason return nil default: return fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, report.Kind) } } func (t *SmokeTester) finishHTTP( ctx context.Context, report *domain.SmokeTestReport, finish func(error) (*domain.SmokeTestReport, error), ) (*domain.SmokeTestReport, error) { if contextErr := ctx.Err(); contextErr != nil && isContextFailure(report.Failure) { report.Completed = false report.Failure = generation.ClassifyFailure(contextErr) return finish(contextErr) } report.Completed = true return finish(nil) } func (t *SmokeTester) recordHTTPAttempt( report *domain.SmokeTestReport, initialEndpoint string, evidence llm.HTTPExecutionEvidence, startedAt time.Time, duration time.Duration, failure domain.ExecutionFailureKind, credentialsAccepted *bool, ) { if !evidence.DispatchInvoked { return } reportDuration := duration if report.Timing.RequestDuration == nil { report.Timing.RequestDuration = &reportDuration } else { *report.Timing.RequestDuration += reportDuration } attempt := domain.SmokeTestAttempt{ Dispatched: true, InitialEndpoint: initialEndpoint, Endpoint: evidence.Endpoint, Failure: failure, Usage: domain.ObservedTokenUsage{}, Facts: httpExecutionFacts(evidence), Timing: domain.SmokeTestTiming{ StartedAt: startedAt, Duration: duration, RequestDuration: &duration, }, } if evidence.HTTPStatus != nil { status := *evidence.HTTPStatus attempt.HTTPStatus = &status } attempt.Facts.CredentialsAccepted = cloneOptionalBool(credentialsAccepted) report.Attempts = append(report.Attempts, attempt) } func httpExecutionFacts(evidence llm.HTTPExecutionEvidence) domain.SmokeTestFacts { var facts domain.SmokeTestFacts if evidence.ResponseReceived || evidence.DispatchInvoked { responseReceived := evidence.ResponseReceived facts.ResponseReceived = &responseReceived } if evidence.DispatchInvoked { reachable := evidence.ResponseReceived facts.EndpointReachable = &reachable } if evidence.RequestObserved { credentialsSupplied := evidence.CredentialsSupplied facts.CredentialsSupplied = &credentialsSupplied } return facts } func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) { cloned, err := domain.CloneSmokeTestReport(domain.SmokeTestReport{Target: target}) if err != nil { return domain.SmokeTestTarget{}, fmt.Errorf("clone smoke-test target settings: %w", err) } target = cloned.Target target.EffectiveSettings.APIKey = "" target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint) if err != nil { return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest) } target.EffectiveSettings.Endpoint, err = endpointWithoutUserInfo(target.EffectiveSettings.Endpoint) if err != nil { return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved execution endpoint", ErrInvalidRequest) } return target, nil } func cloneOptionalBool(value *bool) *bool { if value == nil { return nil } cloned := *value return &cloned } func isContextFailure(failure domain.ExecutionFailureKind) bool { return failure == domain.ExecutionFailureCanceled || failure == domain.ExecutionFailureDeadline } func nonnegativeDuration(duration time.Duration) time.Duration { if duration < 0 { return 0 } return duration }