promptkit/internal/usecase/smoke_tester.go

package usecase

import (
	"context"
	"errors"
	"fmt"
	"time"

	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
	"gitea.maximumdirect.net/eric/promptkit/internal/generation"
	"gitea.maximumdirect.net/eric/promptkit/internal/llm"
	"gitea.maximumdirect.net/eric/promptkit/internal/observation"
)

// SmokeTester coordinates internal diagnostic operations over captured
// selection collaborators, shared generation, and optional built-in HTTP.
type SmokeTester struct {
	runner     *Runner
	httpClient smokeHTTPGetter
	now        func() time.Time
}

// NewSmokeTester creates an internal diagnostic coordinator. A nil HTTP client
// leaves HTTP operations unsupported rather than falling back to another client.
func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester {
	var getter smokeHTTPGetter
	if httpClient != nil {
		getter = httpClient
	}
	return &SmokeTester{runner: runner, httpClient: getter, now: time.Now}
}

func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester {
	var getter smokeHTTPGetter
	if httpClient != nil {
		getter = httpClient
	}
	return &SmokeTester{runner: runner, httpClient: getter, now: now}
}

// Run selects and executes one diagnostic, retaining only factual results.
func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
	if t == nil || t.runner == nil {
		return nil, fmt.Errorf("%w: smoke tester is not configured", ErrInvalidRequest)
	}
	ctx = observation.WithoutExecutionState(ctx)
	now := t.now
	if now == nil {
		now = time.Now
	}
	startedAt := now().UTC()
	report := &domain.SmokeTestReport{
		Kind: request.Kind,
		Timing: domain.SmokeTestTiming{
			StartedAt: startedAt,
		},
	}
	finish := func(err error) (*domain.SmokeTestReport, error) {
		report.Timing.Duration = nonnegativeDuration(now().UTC().Sub(startedAt))
		for _, attempt := range report.Attempts {
			if attempt.Dispatched {
				return report, err
			}
		}
		report.Usage = noDispatchUsage()
		return report, err
	}

	selection, err := t.runner.resolveSmokeTestTarget(ctx, request)
	if err != nil {
		if contextErr := ctx.Err(); contextErr != nil {
			report.Failure = generation.ClassifyFailure(contextErr)
			return finish(contextErr)
		}
		return nil, err
	}
	report.Target, err = smokeReportTarget(selection.target)
	if err != nil {
		return nil, err
	}
	if isInferenceSmokeTest(request.Kind) {
		// The configured base is not evidence of an HTTP request destination.
		// Built-in generation supplies its composed URL after preparation.
		report.Target.Endpoint = ""
		report.Supported = true
		report.Applicable = boolPointer(true)
	} else if err := t.setSupport(report, selection); err != nil {
		return nil, err
	}
	if contextErr := ctx.Err(); contextErr != nil {
		report.Failure = generation.ClassifyFailure(contextErr)
		return finish(contextErr)
	}
	if !report.Supported || (report.Applicable != nil && !*report.Applicable) {
		report.Completed = true
		return finish(nil)
	}

	switch request.Kind {
	case domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput:
		return t.runInference(ctx, request, selection, report, now, finish)
	case domain.SmokeTestReachability:
		requestStartedAt := now().UTC()
		outcome, err := (reachabilityProbe{client: t.httpClient}).run(
			ctx,
			selection.target.Endpoint,
			selection.target.EffectiveSettings.TimeoutSeconds,
		)
		requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
		if err != nil {
			if contextErr := ctx.Err(); contextErr != nil {
				report.Failure = generation.ClassifyFailure(contextErr)
				return finish(contextErr)
			}
			return nil, err
		}
		report.Supported = outcome.supported
		report.Reason = outcome.reason
		report.Failure = outcome.failure
		report.Facts = httpExecutionFacts(outcome.evidence)
		if outcome.reachable != nil {
			reachable := *outcome.reachable
			report.Facts.EndpointReachable = &reachable
		}
		t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil)
		return t.finishHTTP(ctx, report, finish)
	case domain.SmokeTestAuthentication:
		requestStartedAt := now().UTC()
		outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target)
		requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
		if err != nil {
			if contextErr := ctx.Err(); contextErr != nil {
				report.Failure = generation.ClassifyFailure(contextErr)
				return finish(contextErr)
			}
			return nil, err
		}
		report.Supported = outcome.supported
		report.Applicable = cloneOptionalBool(outcome.applicable)
		report.Reason = outcome.reason
		report.Failure = outcome.failure
		report.Facts = httpExecutionFacts(outcome.evidence)
		report.Facts.CredentialsAccepted = cloneOptionalBool(outcome.credentialsAccepted)
		t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, outcome.credentialsAccepted)
		return t.finishHTTP(ctx, report, finish)
	default:
		return nil, fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, request.Kind)
	}
}

type smokeEvidenceGenerator interface {
	GenerateWithEvidence(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error, generation.GenerationEvidence)
}

func (t *SmokeTester) runInference(
	ctx context.Context,
	request domain.SmokeTestRequest,
	selection *smokeTestSelection,
	report *domain.SmokeTestReport,
	now func() time.Time,
	finish func(error) (*domain.SmokeTestReport, error),
) (*domain.SmokeTestReport, error) {
	settings := selection.target.EffectiveSettings
	if err := validateAPIKey(settings.APIKeyEnv, settings.APIKey, settings.APIKeyRequired); err != nil {
		if contextErr := ctx.Err(); contextErr != nil {
			report.Failure = generation.ClassifyFailure(contextErr)
			return finish(contextErr)
		}
		if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) {
			report.Reason = domain.SmokeTestReasonRequiredCredentialMissing
			report.Failure = domain.ExecutionFailureInvalidRequest
			report.Completed = true
			return finish(nil)
		}
		return nil, err
	}

	inferenceCase, err := prepareSmokeInferenceCase(ctx, request.Kind, selection.target, selection.transientRetries)
	if err != nil {
		if contextErr := ctx.Err(); contextErr != nil {
			report.Failure = generation.ClassifyFailure(contextErr)
			return finish(contextErr)
		}
		return nil, err
	}
	generator, ok := t.runner.llm.(smokeEvidenceGenerator)
	if !ok {
		return nil, fmt.Errorf("smoke-test generation evidence is unavailable")
	}

	release, err := t.runner.admitRun(ctx, selection.target.BackendID)
	if err != nil {
		if contextErr := ctx.Err(); contextErr != nil {
			report.Failure = generation.ClassifyFailure(contextErr)
			return finish(contextErr)
		}
		var capacityErr *CapacityError
		if errors.As(err, &capacityErr) {
			report.Failure = domain.ExecutionFailureCapacity
			report.Completed = true
			return finish(nil)
		}
		return nil, fmt.Errorf("admit smoke-test generation: %w", err)
	}
	defer release()

	response, generationErr, evidence := generator.GenerateWithEvidence(ctx, inferenceCase.request)
	t.recordGenerationEvidence(report, evidence)
	if contextErr := ctx.Err(); contextErr != nil {
		report.Failure = generation.ClassifyFailure(contextErr)
		return finish(contextErr)
	}
	if generationErr != nil {
		report.Failure = generation.ClassifyFailure(generationErr)
		report.Completed = true
		return finish(nil)
	}
	if response == nil {
		report.Failure = generation.ClassifyFailure(llm.ErrMalformedResponse)
		report.Completed = true
		return finish(nil)
	}

	evaluationStartedAt := now().UTC()
	evaluatedFacts, evaluationErr := inferenceCase.evaluate(ctx, response)
	evaluationDuration := nonnegativeDuration(now().UTC().Sub(evaluationStartedAt))
	report.Timing.EvaluationDuration = &evaluationDuration
	mergeEvaluatedSmokeFacts(&report.Facts, evaluatedFacts)
	if contextErr := ctx.Err(); contextErr != nil {
		report.Failure = generation.ClassifyFailure(contextErr)
		return finish(contextErr)
	}
	if evaluationErr != nil {
		return nil, fmt.Errorf("evaluate smoke-test response: %w", evaluationErr)
	}
	report.Completed = true
	report.Failure = domain.ExecutionFailureNone
	if request.Kind == domain.SmokeTestStructuredOutput &&
		((report.Facts.ValidJSON != nil && !*report.Facts.ValidJSON) ||
			(report.Facts.SchemaMatched != nil && !*report.Facts.SchemaMatched)) {
		report.Failure = domain.ExecutionFailureValidation
	}
	return finish(nil)
}

func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
	report.Target.Endpoint = evidence.InitialEndpoint
	report.Usage = evidence.Usage
	var capacityWait, requestDuration time.Duration
	for _, attemptEvidence := range evidence.Attempts {
		capacityWait += attemptEvidence.CapacityWait
		attempt := domain.SmokeTestAttempt{
			Dispatched:      attemptEvidence.Dispatched,
			InitialEndpoint: evidence.InitialEndpoint,
			Endpoint:        attemptEvidence.HTTP.Endpoint,
			Facts:           generationAttemptFacts(attemptEvidence),
			Usage:           attemptEvidence.Usage,
			Failure:         attemptEvidence.Failure,
			Timing: domain.SmokeTestTiming{
				StartedAt:    attemptEvidence.StartedAt,
				Duration:     nonnegativeDuration(attemptEvidence.CapacityWait + attemptEvidence.RequestDuration),
				CapacityWait: durationPointer(attemptEvidence.CapacityWait),
			},
		}
		if attemptEvidence.Dispatched {
			requestDuration += attemptEvidence.RequestDuration
			attempt.Timing.RequestDuration = durationPointer(attemptEvidence.RequestDuration)
		}
		if attemptEvidence.HTTP.HTTPStatus != nil {
			status := *attemptEvidence.HTTP.HTTPStatus
			attempt.HTTPStatus = &status
		}
		report.Attempts = append(report.Attempts, attempt)
		if attemptEvidence.Dispatched {
			report.Facts = generationAttemptFacts(attemptEvidence)
		}
	}
	if len(evidence.Attempts) > 0 {
		report.Timing.CapacityWait = durationPointer(capacityWait)
	}
	if requestDurationKnown(evidence.Attempts) {
		report.Timing.RequestDuration = durationPointer(requestDuration)
	}
	if len(evidence.RetryDelays) > 0 {
		var retryDelay time.Duration
		for _, delay := range evidence.RetryDelays {
			retryDelay += delay
		}
		report.Timing.RetryDelay = durationPointer(retryDelay)
	}
}

func generationAttemptFacts(evidence generation.AttemptEvidence) domain.SmokeTestFacts {
	facts := httpExecutionFacts(evidence.HTTP)
	if evidence.Dispatched {
		responseReceived := evidence.ResponseReceived
		generationCompleted := evidence.GenerationCompleted
		facts.ResponseReceived = &responseReceived
		facts.GenerationCompleted = &generationCompleted
	}
	facts.StructuredOutputRequested = cloneOptionalBool(evidence.StructuredOutputRequested)
	return facts
}

func mergeEvaluatedSmokeFacts(target *domain.SmokeTestFacts, evaluated domain.SmokeTestFacts) {
	if evaluated.TextReturned != nil {
		target.TextReturned = cloneOptionalBool(evaluated.TextReturned)
	}
	if evaluated.ExpectedTextMatched != nil {
		target.ExpectedTextMatched = cloneOptionalBool(evaluated.ExpectedTextMatched)
	}
	if evaluated.ValidJSON != nil {
		target.ValidJSON = cloneOptionalBool(evaluated.ValidJSON)
	}
	if evaluated.SchemaMatched != nil {
		target.SchemaMatched = cloneOptionalBool(evaluated.SchemaMatched)
	}
}

func noDispatchUsage() domain.ObservedTokenUsage {
	return (observation.UsageAccumulator{}).Observed()
}

func durationPointer(value time.Duration) *time.Duration {
	cloned := value
	return &cloned
}

func requestDurationKnown(attempts []generation.AttemptEvidence) bool {
	for _, attempt := range attempts {
		if attempt.Dispatched {
			return true
		}
	}
	return false
}

func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error {
	switch report.Kind {
	case domain.SmokeTestReachability:
		if t.httpClient == nil {
			report.Reason = domain.SmokeTestReasonHTTPDiagnosticsUnavailable
			return nil
		}
		report.Supported = true
		report.Applicable = boolPointer(true)
		return nil
	case domain.SmokeTestAuthentication:
		outcome, err := (authenticationProbe{client: t.httpClient}).support(selection.backend)
		if err != nil {
			return err
		}
		report.Supported = outcome.supported
		report.Applicable = cloneOptionalBool(outcome.applicable)
		report.Reason = outcome.reason
		return nil
	default:
		return fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, report.Kind)
	}
}

func (t *SmokeTester) finishHTTP(
	ctx context.Context,
	report *domain.SmokeTestReport,
	finish func(error) (*domain.SmokeTestReport, error),
) (*domain.SmokeTestReport, error) {
	if contextErr := ctx.Err(); contextErr != nil && isContextFailure(report.Failure) {
		report.Completed = false
		report.Failure = generation.ClassifyFailure(contextErr)
		return finish(contextErr)
	}
	report.Completed = true
	return finish(nil)
}

func (t *SmokeTester) recordHTTPAttempt(
	report *domain.SmokeTestReport,
	initialEndpoint string,
	evidence llm.HTTPExecutionEvidence,
	startedAt time.Time,
	duration time.Duration,
	failure domain.ExecutionFailureKind,
	credentialsAccepted *bool,
) {
	if !evidence.DispatchInvoked {
		return
	}
	reportDuration := duration
	if report.Timing.RequestDuration == nil {
		report.Timing.RequestDuration = &reportDuration
	} else {
		*report.Timing.RequestDuration += reportDuration
	}
	attempt := domain.SmokeTestAttempt{
		Dispatched:      true,
		InitialEndpoint: initialEndpoint,
		Endpoint:        evidence.Endpoint,
		Failure:         failure,
		Usage:           domain.ObservedTokenUsage{},
		Facts:           httpExecutionFacts(evidence),
		Timing: domain.SmokeTestTiming{
			StartedAt:       startedAt,
			Duration:        duration,
			RequestDuration: &duration,
		},
	}
	if evidence.HTTPStatus != nil {
		status := *evidence.HTTPStatus
		attempt.HTTPStatus = &status
	}
	attempt.Facts.CredentialsAccepted = cloneOptionalBool(credentialsAccepted)
	report.Attempts = append(report.Attempts, attempt)
}

func httpExecutionFacts(evidence llm.HTTPExecutionEvidence) domain.SmokeTestFacts {
	var facts domain.SmokeTestFacts
	if evidence.ResponseReceived || evidence.DispatchInvoked {
		responseReceived := evidence.ResponseReceived
		facts.ResponseReceived = &responseReceived
	}
	if evidence.DispatchInvoked {
		reachable := evidence.ResponseReceived
		facts.EndpointReachable = &reachable
	}
	if evidence.RequestObserved {
		credentialsSupplied := evidence.CredentialsSupplied
		facts.CredentialsSupplied = &credentialsSupplied
	}
	return facts
}

func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) {
	cloned, err := domain.CloneSmokeTestReport(domain.SmokeTestReport{Target: target})
	if err != nil {
		return domain.SmokeTestTarget{}, fmt.Errorf("clone smoke-test target settings: %w", err)
	}
	target = cloned.Target
	target.EffectiveSettings.APIKey = ""
	target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint)
	if err != nil {
		return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest)
	}
	target.EffectiveSettings.Endpoint, err = endpointWithoutUserInfo(target.EffectiveSettings.Endpoint)
	if err != nil {
		return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved execution endpoint", ErrInvalidRequest)
	}
	return target, nil
}

func cloneOptionalBool(value *bool) *bool {
	if value == nil {
		return nil
	}
	cloned := *value
	return &cloned
}

func isContextFailure(failure domain.ExecutionFailureKind) bool {
	return failure == domain.ExecutionFailureCanceled || failure == domain.ExecutionFailureDeadline
}

func nonnegativeDuration(duration time.Duration) time.Duration {
	if duration < 0 {
		return 0
	}
	return duration
}