eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate inference smoke diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-13 18:09:20 +00:00
parent f0182fb2c9
commit b268c55783
7 changed files with 624 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
internal/usecase/smoke_tester.go
View File

@@ -2,6 +2,7 @@ package usecase
import (
"context"
"errors"
"fmt"
"time"
@@ -71,7 +72,10 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
if err != nil {
return nil, err
}
if err := t.setSupport(report, selection); err != nil {
if isInferenceSmokeTest(request.Kind) {
report.Supported = true
report.Applicable = boolPointer(true)
} else if err := t.setSupport(report, selection); err != nil {
return nil, err
}
if contextErr := ctx.Err(); contextErr != nil {
@@ -83,9 +87,11 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
return finish(nil)
}
requestStartedAt := now().UTC()
switch request.Kind {
case domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput:
return t.runInference(ctx, request, selection, report, now, finish)
case domain.SmokeTestReachability:
requestStartedAt := now().UTC()
outcome, err := (reachabilityProbe{client: t.httpClient}).run(
ctx,
selection.target.Endpoint,
@@ -110,6 +116,7 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
t.recordHTTPAttempt(report, report.Target.Endpoint, outcome.evidence, requestStartedAt, requestDuration, outcome.failure, nil)
return t.finishHTTP(ctx, report, finish)
case domain.SmokeTestAuthentication:
requestStartedAt := now().UTC()
outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target)
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
if err != nil {
@@ -132,6 +139,195 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
}
}
type smokeEvidenceGenerator interface {
GenerateWithEvidence(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error, generation.GenerationEvidence)
}
func (t *SmokeTester) runInference(
ctx context.Context,
request domain.SmokeTestRequest,
selection *smokeTestSelection,
report *domain.SmokeTestReport,
now func() time.Time,
finish func(error) (*domain.SmokeTestReport, error),
) (*domain.SmokeTestReport, error) {
settings := selection.target.EffectiveSettings
if err := validateAPIKey(settings.APIKeyEnv, settings.APIKey, settings.APIKeyRequired); err != nil {
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
return finish(contextErr)
}
if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) {
report.Reason = domain.SmokeTestReasonRequiredCredentialMissing
report.Failure = domain.ExecutionFailureInvalidRequest
report.Usage = noDispatchUsage()
report.Completed = true
return finish(nil)
}
return nil, err
}
inferenceCase, err := prepareSmokeInferenceCase(ctx, request.Kind, selection.target, selection.transientRetries)
if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
return finish(contextErr)
}
return nil, err
}
generator, ok := t.runner.llm.(smokeEvidenceGenerator)
if !ok {
return nil, fmt.Errorf("smoke-test generation evidence is unavailable")
}
release, err := t.runner.admitRun(ctx, selection.target.BackendID)
if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
report.Usage = noDispatchUsage()
return finish(contextErr)
}
var capacityErr *CapacityError
if errors.As(err, &capacityErr) {
report.Failure = domain.ExecutionFailureCapacity
report.Usage = noDispatchUsage()
report.Completed = true
return finish(nil)
}
return nil, fmt.Errorf("admit smoke-test generation: %w", err)
}
defer release()
response, generationErr, evidence := generator.GenerateWithEvidence(ctx, inferenceCase.request)
t.recordGenerationEvidence(report, evidence)
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
return finish(contextErr)
}
if generationErr != nil {
report.Failure = generation.ClassifyFailure(generationErr)
report.Completed = true
return finish(nil)
}
if response == nil {
report.Failure = generation.ClassifyFailure(llm.ErrMalformedResponse)
report.Completed = true
return finish(nil)
}
evaluationStartedAt := now().UTC()
evaluatedFacts, evaluationErr := inferenceCase.evaluate(ctx, response)
evaluationDuration := nonnegativeDuration(now().UTC().Sub(evaluationStartedAt))
report.Timing.EvaluationDuration = &evaluationDuration
mergeEvaluatedSmokeFacts(&report.Facts, evaluatedFacts)
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
return finish(contextErr)
}
if evaluationErr != nil {
return nil, fmt.Errorf("evaluate smoke-test response: %w", evaluationErr)
}
report.Completed = true
report.Failure = domain.ExecutionFailureNone
if request.Kind == domain.SmokeTestStructuredOutput &&
((report.Facts.ValidJSON != nil && !*report.Facts.ValidJSON) ||
(report.Facts.SchemaMatched != nil && !*report.Facts.SchemaMatched)) {
report.Failure = domain.ExecutionFailureValidation
}
return finish(nil)
}
func (t *SmokeTester) recordGenerationEvidence(report *domain.SmokeTestReport, evidence generation.GenerationEvidence) {
report.Usage = evidence.Usage
var capacityWait, requestDuration time.Duration
for _, attemptEvidence := range evidence.Attempts {
capacityWait += attemptEvidence.CapacityWait
attempt := domain.SmokeTestAttempt{
Dispatched: attemptEvidence.Dispatched,
InitialEndpoint: report.Target.Endpoint,
Endpoint: attemptEvidence.HTTP.Endpoint,
Facts: generationAttemptFacts(attemptEvidence),
Usage: attemptEvidence.Usage,
Failure: attemptEvidence.Failure,
Timing: domain.SmokeTestTiming{
StartedAt: attemptEvidence.StartedAt,
Duration: nonnegativeDuration(attemptEvidence.CapacityWait + attemptEvidence.RequestDuration),
CapacityWait: durationPointer(attemptEvidence.CapacityWait),
},
}
if attemptEvidence.Dispatched {
requestDuration += attemptEvidence.RequestDuration
attempt.Timing.RequestDuration = durationPointer(attemptEvidence.RequestDuration)
}
if attemptEvidence.HTTP.HTTPStatus != nil {
status := *attemptEvidence.HTTP.HTTPStatus
attempt.HTTPStatus = &status
}
report.Attempts = append(report.Attempts, attempt)
if attemptEvidence.Dispatched {
report.Facts = generationAttemptFacts(attemptEvidence)
}
}
if len(evidence.Attempts) > 0 {
report.Timing.CapacityWait = durationPointer(capacityWait)
}
if requestDurationKnown(evidence.Attempts) {
report.Timing.RequestDuration = durationPointer(requestDuration)
}
if len(evidence.RetryDelays) > 0 {
var retryDelay time.Duration
for _, delay := range evidence.RetryDelays {
retryDelay += delay
}
report.Timing.RetryDelay = durationPointer(retryDelay)
}
}
func generationAttemptFacts(evidence generation.AttemptEvidence) domain.SmokeTestFacts {
facts := httpExecutionFacts(evidence.HTTP)
if evidence.Dispatched {
responseReceived := evidence.ResponseReceived
generationCompleted := evidence.GenerationCompleted
facts.ResponseReceived = &responseReceived
facts.GenerationCompleted = &generationCompleted
}
facts.StructuredOutputRequested = cloneOptionalBool(evidence.StructuredOutputRequested)
return facts
}
func mergeEvaluatedSmokeFacts(target *domain.SmokeTestFacts, evaluated domain.SmokeTestFacts) {
if evaluated.TextReturned != nil {
target.TextReturned = cloneOptionalBool(evaluated.TextReturned)
}
if evaluated.ExpectedTextMatched != nil {
target.ExpectedTextMatched = cloneOptionalBool(evaluated.ExpectedTextMatched)
}
if evaluated.ValidJSON != nil {
target.ValidJSON = cloneOptionalBool(evaluated.ValidJSON)
}
if evaluated.SchemaMatched != nil {
target.SchemaMatched = cloneOptionalBool(evaluated.SchemaMatched)
}
}
func noDispatchUsage() domain.ObservedTokenUsage {
return (observation.UsageAccumulator{}).Observed()
}
func durationPointer(value time.Duration) *time.Duration {
cloned := value
return &cloned
}
func requestDurationKnown(attempts []generation.AttemptEvidence) bool {
for _, attempt := range attempts {
if attempt.Dispatched {
return true
}
}
return false
}
func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smokeTestSelection) error {
switch report.Kind {
case domain.SmokeTestReachability: