Assemble HTTP smoke-test reports
This commit is contained in:
264
internal/usecase/smoke_tester.go
Normal file
264
internal/usecase/smoke_tester.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package usecase
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
// SmokeTester coordinates internal HTTP smoke-test operations over captured
|
||||
// selection collaborators. Inference operations are added separately.
|
||||
type SmokeTester struct {
|
||||
runner *Runner
|
||||
httpClient smokeHTTPGetter
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// NewSmokeTester creates an internal diagnostic coordinator. A nil HTTP client
|
||||
// leaves HTTP operations unsupported rather than falling back to another client.
|
||||
func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester {
|
||||
var getter smokeHTTPGetter
|
||||
if httpClient != nil {
|
||||
getter = httpClient
|
||||
}
|
||||
return &SmokeTester{runner: runner, httpClient: getter, now: time.Now}
|
||||
}
|
||||
|
||||
func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester {
|
||||
var getter smokeHTTPGetter
|
||||
if httpClient != nil {
|
||||
getter = httpClient
|
||||
}
|
||||
return &SmokeTester{runner: runner, httpClient: getter, now: now}
|
||||
}
|
||||
|
||||
// Run selects and executes one HTTP diagnostic, retaining only factual results.
|
||||
func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest) (*domain.SmokeTestReport, error) {
|
||||
if t == nil || t.runner == nil {
|
||||
return nil, fmt.Errorf("%w: smoke tester is not configured", ErrInvalidRequest)
|
||||
}
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
now := t.now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
startedAt := now().UTC()
|
||||
report := &domain.SmokeTestReport{
|
||||
Kind: request.Kind,
|
||||
Timing: domain.SmokeTestTiming{
|
||||
StartedAt: startedAt,
|
||||
},
|
||||
}
|
||||
finish := func(err error) (*domain.SmokeTestReport, error) {
|
||||
report.Timing.Duration = nonnegativeDuration(now().UTC().Sub(startedAt))
|
||||
return report, err
|
||||
}
|
||||
|
||||
selection, err := t.runner.resolveSmokeTestTarget(ctx, request)
|
||||
if err != nil {
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
report.Failure = generation.ClassifyFailure(contextErr)
|
||||
return finish(contextErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
report.Target, err = smokeReportTarget(selection.target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := t.setSupport(report, selection); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
report.Failure = generation.ClassifyFailure(contextErr)
|
||||
return finish(contextErr)
|
||||
}
|
||||
if !report.Supported || (report.Applicable != nil && !*report.Applicable) {
|
||||
report.Completed = true
|
||||
return finish(nil)
|
||||
}
|
||||
|
||||
requestStartedAt := now().UTC()
|
||||
switch request.Kind {
|
||||
case domain.SmokeTestReachability:
|
||||
outcome, err := (reachabilityProbe{client: t.httpClient}).run(
|
||||
ctx,
|
||||
selection.target.Endpoint,
|
||||
selection.target.EffectiveSettings.TimeoutSeconds,
|
||||
)
|
||||
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
|
||||
if err != nil {
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
report.Failure = generation.ClassifyFailure(contextErr)
|
||||
return finish(contextErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
report.Supported = outcome.supported
|
||||
report.Reason = outcome.reason
|
||||
report.Failure = outcome.failure
|
||||
report.Facts = httpExecutionFacts(outcome.evidence)
|
||||
if outcome.reachable != nil {
|
||||
reachable := *outcome.reachable
|
||||
report.Facts.EndpointReachable = &reachable
|
||||
}
|
||||
t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil)
|
||||
return t.finishHTTP(ctx, report, finish)
|
||||
case domain.SmokeTestAuthentication:
|
||||
outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target)
|
||||
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
|
||||
if err != nil {
|
||||
if contextErr := ctx.Err(); contextErr != nil {
|
||||
report.Failure = generation.ClassifyFailure(contextErr)
|
||||
return finish(contextErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
report.Supported = outcome.supported
|
||||
report.Applicable = cloneOptionalBool(outcome.applicable)
|
||||
report.Reason = outcome.reason
|
||||
report.Failure = outcome.failure
|
||||
report.Facts = httpExecutionFacts(outcome.evidence)
|
||||
report.Facts.CredentialsAccepted = cloneOptionalBool(outcome.credentialsAccepted)
|
||||
t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, outcome.credentialsAccepted)
|
||||
return t.finishHTTP(ctx, report, finish)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, request.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error {
|
||||
switch report.Kind {
|
||||
case domain.SmokeTestReachability:
|
||||
if t.httpClient == nil {
|
||||
report.Reason = domain.SmokeTestReasonHTTPDiagnosticsUnavailable
|
||||
return nil
|
||||
}
|
||||
report.Supported = true
|
||||
report.Applicable = boolPointer(true)
|
||||
return nil
|
||||
case domain.SmokeTestAuthentication:
|
||||
outcome, err := (authenticationProbe{client: t.httpClient}).support(selection.backend)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Supported = outcome.supported
|
||||
report.Applicable = cloneOptionalBool(outcome.applicable)
|
||||
report.Reason = outcome.reason
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("%w: HTTP smoke tester cannot run kind %q", ErrInvalidRequest, report.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *SmokeTester) finishHTTP(
|
||||
ctx context.Context,
|
||||
report *domain.SmokeTestReport,
|
||||
finish func(error) (*domain.SmokeTestReport, error),
|
||||
) (*domain.SmokeTestReport, error) {
|
||||
if contextErr := ctx.Err(); contextErr != nil && isContextFailure(report.Failure) {
|
||||
report.Completed = false
|
||||
report.Failure = generation.ClassifyFailure(contextErr)
|
||||
return finish(contextErr)
|
||||
}
|
||||
report.Completed = true
|
||||
return finish(nil)
|
||||
}
|
||||
|
||||
func (t *SmokeTester) recordHTTPAttempt(
|
||||
report *domain.SmokeTestReport,
|
||||
initialEndpoint string,
|
||||
evidence llm.HTTPExecutionEvidence,
|
||||
startedAt time.Time,
|
||||
duration time.Duration,
|
||||
failure domain.ExecutionFailureKind,
|
||||
credentialsAccepted *bool,
|
||||
) {
|
||||
if !evidence.DispatchInvoked {
|
||||
return
|
||||
}
|
||||
reportDuration := duration
|
||||
if report.Timing.RequestDuration == nil {
|
||||
report.Timing.RequestDuration = &reportDuration
|
||||
} else {
|
||||
*report.Timing.RequestDuration += reportDuration
|
||||
}
|
||||
attempt := domain.SmokeTestAttempt{
|
||||
Dispatched: true,
|
||||
InitialEndpoint: initialEndpoint,
|
||||
Endpoint: evidence.Endpoint,
|
||||
Failure: failure,
|
||||
Usage: domain.ObservedTokenUsage{},
|
||||
Facts: httpExecutionFacts(evidence),
|
||||
Timing: domain.SmokeTestTiming{
|
||||
StartedAt: startedAt,
|
||||
Duration: duration,
|
||||
RequestDuration: &duration,
|
||||
},
|
||||
}
|
||||
if evidence.HTTPStatus != nil {
|
||||
status := *evidence.HTTPStatus
|
||||
attempt.HTTPStatus = &status
|
||||
}
|
||||
attempt.Facts.CredentialsAccepted = cloneOptionalBool(credentialsAccepted)
|
||||
report.Attempts = append(report.Attempts, attempt)
|
||||
}
|
||||
|
||||
func httpExecutionFacts(evidence llm.HTTPExecutionEvidence) domain.SmokeTestFacts {
|
||||
var facts domain.SmokeTestFacts
|
||||
if evidence.ResponseReceived || evidence.DispatchInvoked {
|
||||
responseReceived := evidence.ResponseReceived
|
||||
facts.ResponseReceived = &responseReceived
|
||||
}
|
||||
if evidence.DispatchInvoked {
|
||||
reachable := evidence.ResponseReceived
|
||||
facts.EndpointReachable = &reachable
|
||||
}
|
||||
if evidence.RequestObserved {
|
||||
credentialsSupplied := evidence.CredentialsSupplied
|
||||
facts.CredentialsSupplied = &credentialsSupplied
|
||||
}
|
||||
return facts
|
||||
}
|
||||
|
||||
func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) {
|
||||
cloned, err := domain.CloneSmokeTestReport(domain.SmokeTestReport{Target: target})
|
||||
if err != nil {
|
||||
return domain.SmokeTestTarget{}, fmt.Errorf("clone smoke-test target settings: %w", err)
|
||||
}
|
||||
target = cloned.Target
|
||||
target.EffectiveSettings.APIKey = ""
|
||||
target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint)
|
||||
if err != nil {
|
||||
return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest)
|
||||
}
|
||||
target.EffectiveSettings.Endpoint, err = endpointWithoutUserInfo(target.EffectiveSettings.Endpoint)
|
||||
if err != nil {
|
||||
return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved execution endpoint", ErrInvalidRequest)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func cloneOptionalBool(value *bool) *bool {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func isContextFailure(failure domain.ExecutionFailureKind) bool {
|
||||
return failure == domain.ExecutionFailureCanceled || failure == domain.ExecutionFailureDeadline
|
||||
}
|
||||
|
||||
func nonnegativeDuration(duration time.Duration) time.Duration {
|
||||
if duration < 0 {
|
||||
return 0
|
||||
}
|
||||
return duration
|
||||
}
|
||||
Reference in New Issue
Block a user