eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Report known zero usage when smoke tests do not dispatch

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-13 20:32:24 +00:00
parent 116bd0e39b
commit 5ea6f06a6d
3 changed files with 31 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/usecase/smoke_tester.go
View File

@@ -57,6 +57,12 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
}
finish := func(err error) (*domain.SmokeTestReport, error) {
report.Timing.Duration = nonnegativeDuration(now().UTC().Sub(startedAt))
for _, attempt := range report.Attempts {
if attempt.Dispatched {
return report, err
}
}
report.Usage = noDispatchUsage()
return report, err
}
@@ -163,7 +169,6 @@ func (t *SmokeTester) runInference(
if errors.Is(err, ErrAPIKeyRequired) || errors.Is(err, ErrAPIKeyEnvMissing) {
report.Reason = domain.SmokeTestReasonRequiredCredentialMissing
report.Failure = domain.ExecutionFailureInvalidRequest
report.Usage = noDispatchUsage()
report.Completed = true
return finish(nil)
}
@@ -187,13 +192,11 @@ func (t *SmokeTester) runInference(
if err != nil {
if contextErr := ctx.Err(); contextErr != nil {
report.Failure = generation.ClassifyFailure(contextErr)
report.Usage = noDispatchUsage()
return finish(contextErr)
}
var capacityErr *CapacityError
if errors.As(err, &capacityErr) {
report.Failure = domain.ExecutionFailureCapacity
report.Usage = noDispatchUsage()
report.Completed = true
return finish(nil)
}

23
internal/usecase/smoke_tester_test.go
View File

@@ -184,7 +184,8 @@ func TestSmokeTesterReportsUnsupportedAndInapplicableAuthentication(t *testing.T
t.Fatal(err)
}
if report.Supported != tc.supported || report.Reason != tc.reason || !report.Completed ||
!sameBool(report.Applicable, tc.applicable) || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) {
!sameBool(report.Applicable, tc.applicable) || len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) ||
report.Usage != noDispatchUsage() {
t.Fatalf("report = %+v", report)
}
})
@@ -201,7 +202,7 @@ func TestSmokeTesterReportsReachabilityUnavailableWithoutHTTPClient(t *testing.T
}
if report.Supported || report.Applicable != nil || !report.Completed ||
report.Reason != domain.SmokeTestReasonHTTPDiagnosticsUnavailable ||
len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) {
len(report.Attempts) != 0 || report.Facts != (domain.SmokeTestFacts{}) || report.Usage != noDispatchUsage() {
t.Fatalf("unsupported reachability report = %+v", report)
}
}
@@ -226,14 +227,16 @@ func TestSmokeTesterSelectionAndCancellationDoNotDispatch(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
report, err := tester.Run(ctx, domain.SmokeTestRequest{
Kind: domain.SmokeTestReachability, BackendID: "alpha",
})
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
t.Fatalf("canceled report=%+v error=%v", report, err)
}
if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || calls != 0 {
t.Fatalf("cancellation dispatched or invented facts: %+v calls=%d", report, calls)
for _, kind := range []domain.SmokeTestKind{domain.SmokeTestReachability, domain.SmokeTestTextInference} {
report, err := tester.Run(ctx, domain.SmokeTestRequest{
Kind: kind, BackendID: "alpha",
})
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != domain.ExecutionFailureCanceled {
t.Fatalf("canceled report=%+v error=%v", report, err)
}
if report.Facts != (domain.SmokeTestFacts{}) || len(report.Attempts) != 0 || calls != 0 || report.Usage != noDispatchUsage() {
t.Fatalf("cancellation dispatched, invented facts, or left usage unknown: %+v calls=%d", report, calls)
}
}
}