From 2e14d973ee2be3f9e8ea3f71fbd962f495e77763 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 17 Oct 2026 18:57:37 +0000 Subject: [PATCH] Coordinate aggregate PromptKit smoke tests --- docs/roadmap/implementation.md | 12 +- integrations/ckprompt/smoke_aggregate_test.go | 159 ++++++++++++++++++ integrations/ckprompt/smoke_command.go | 101 ++++++++++- integrations/ckprompt/smoke_command_test.go | 9 +- 4 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 integrations/ckprompt/smoke_aggregate_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index b323afd..ef72fb8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–11 complete; Stage 12 is next. +Status: Stages 1–12 complete; Stage 13 is next. ## Authority and baseline @@ -624,6 +624,16 @@ stopping, ordered output, and joined error identity. Run focused aggregate tests **Complete when:** `promptkit test all` provides the agreed coordination through the same concrete-operation callback as individual tests. +**Completed:** 2026-10-17. `all` now runs reachability, authentication, text +inference, and structured output sequentially in one invocation, with one input +check and shared state. Each callback receives a concrete kind and a fresh +execution override with copied optional pointers. Reports and per-kind errors +remain ordered; ordinary errors do not stop later tests, while cancellation +stops scheduling and retains earlier results plus joined error identity. +Aggregate tests verify shared lifetimes, seed isolation, continuation, +projection order, and partial cancellation. Focused smoke/report tests and +`GOTOOLCHAIN=local go build ./...` passed in ckprompt. + ## Stage 13: Verify smoke bindings and invocation lifetimes **Primary files:** focused smoke command/adapter tests, reusing existing writer diff --git a/integrations/ckprompt/smoke_aggregate_test.go b/integrations/ckprompt/smoke_aggregate_test.go new file mode 100644 index 0000000..23e0c96 --- /dev/null +++ b/integrations/ckprompt/smoke_aggregate_test.go @@ -0,0 +1,159 @@ +package ckprompt + +import ( + "bytes" + "context" + "errors" + "io" + "reflect" + "testing" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" + "go.yaml.in/yaml/v4" +) + +func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) { + type input struct{ identity *int } + type state struct{ identity *int } + + inputIdentity, stateIdentity := 1, 2 + wantKinds := []promptkit.SmokeTestKind{ + promptkit.SmokeTestReachability, + promptkit.SmokeTestAuthentication, + promptkit.SmokeTestTextInference, + promptkit.SmokeTestStructuredOutput, + } + nativeErr := errors.New("authentication smoke failed") + newInputCalls, checkCalls, newStateCalls := 0, 0, 0 + var gotKinds []promptkit.SmokeTestKind + var executionPointers []*promptkit.ExecutionTargetOverride + var temperaturePointers []*float64 + var retryPointers []*int + var firstInput *input + var sharedState *state + suite := Suite[*input, *state]{ + NewInput: func() *input { + newInputCalls++ + return &input{identity: &inputIdentity} + }, + NewState: func() *state { + newStateCalls++ + return &state{identity: &stateIdentity} + }, + CheckInput: func(_ ckconfig.Selection, value *input) error { + checkCalls++ + firstInput = value + return nil + }, + SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + gotKinds = append(gotKinds, request.Request.Kind) + if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity { + t.Errorf("input/state were not shared: input=%p state=%#v", value, shared) + } + if sharedState == nil { + sharedState = shared + } else if sharedState != shared { + t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared) + } + if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 { + t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request) + } + executionPointers = append(executionPointers, request.Request.Execution) + temperaturePointers = append(temperaturePointers, request.Request.Execution.Temperature) + retryPointers = append(retryPointers, request.Request.TransientRetries) + if len(gotKinds) == 1 { + request.Request.Execution.Model = "builder mutation" + *request.Request.Execution.Temperature = 0.9 + *request.Request.TransientRetries = 11 + } + result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true} + if request.Request.Kind == promptkit.SmokeTestAuthentication { + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr + } + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil + }, + } + app := newPromptTestApp(t, Command(suite)) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{ + "promptkit", "test", "all", "--model=base-model", "--temperature=0.25", "--transient-retries=3", + }, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) { + t.Fatalf("aggregate outcome = %#v", outcome) + } + if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) { + t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds) + } + for index := range executionPointers { + for other := index + 1; other < len(executionPointers); other++ { + if executionPointers[index] == executionPointers[other] || temperaturePointers[index] == temperaturePointers[other] || retryPointers[index] == retryPointers[other] { + t.Fatalf("request pointers were reused between operations %d and %d", index, other) + } + } + } + + var document struct { + Tests []struct { + Kind string `yaml:"kind"` + Report *struct { + Kind string `yaml:"kind"` + } `yaml:"report"` + Error string `yaml:"error"` + } `yaml:"tests"` + } + if err := yaml.Unmarshal(output.Bytes(), &document); err != nil { + t.Fatalf("decode aggregate report %q: %v", output.String(), err) + } + if len(document.Tests) != len(wantKinds) { + t.Fatalf("aggregate entries = %#v", document.Tests) + } + for index, test := range document.Tests { + if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) { + t.Errorf("aggregate entry %d = %#v", index, test) + } + if index == 1 && test.Error != nativeErr.Error() { + t.Errorf("authentication entry error = %q", test.Error) + } + if index != 1 && test.Error != "" { + t.Errorf("unexpected entry error at %d: %q", index, test.Error) + } + } +} + +func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + operationErr := errors.New("second smoke operation failed") + var gotKinds []promptkit.SmokeTestKind + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + gotKinds = append(gotKinds, request.Request.Kind) + if len(gotKinds) == 2 { + cancel() + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr + } + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil + }, + })) + var output bytes.Buffer + outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) { + t.Fatalf("partial aggregate outcome = %#v", outcome) + } + if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) { + t.Fatalf("callbacks after cancellation = %#v", gotKinds) + } + var document struct { + Tests []struct { + Kind string `yaml:"kind"` + } `yaml:"tests"` + } + if err := yaml.Unmarshal(output.Bytes(), &document); err != nil { + t.Fatalf("decode partial aggregate report %q: %v", output.String(), err) + } + if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) { + t.Fatalf("partial aggregate entries = %#v", document.Tests) + } +} diff --git a/integrations/ckprompt/smoke_command.go b/integrations/ckprompt/smoke_command.go index ad0f9d3..b185c73 100644 --- a/integrations/ckprompt/smoke_command.go +++ b/integrations/ckprompt/smoke_command.go @@ -148,10 +148,9 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command { if err := checkSmokeFormat(input.format); err != nil { return err } - if input.kind == "all" { - return errors.New("PromptKit aggregate smoke tests are not available") + if input.kind != "all" { + input.request.Kind = promptkit.SmokeTestKind(input.kind) } - input.request.Kind = promptkit.SmokeTestKind(input.kind) selected, err := prepareSuiteInput(suite, input.selection, input.captured) if err != nil { return err @@ -194,6 +193,9 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], if err := promptKitContextError(invocation.Context); err != nil { return definitionFailure(err, nil) } + if input.kind == "all" { + return runSmokeAggregate(invocation, suite, input, state) + } request := SmokeRequest{Selection: input.selection, Request: input.request} acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state) @@ -218,6 +220,99 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr) } +func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome { + kinds := [...]promptkit.SmokeTestKind{ + promptkit.SmokeTestReachability, + promptkit.SmokeTestAuthentication, + promptkit.SmokeTestTextInference, + promptkit.SmokeTestStructuredOutput, + } + results := make([]smokeResult, 0, len(kinds)) + var diagnostics []commandkit.Diagnostic + var operationErrors []error + + for _, kind := range kinds { + if err := promptKitContextError(invocation.Context); err != nil { + if len(results) == 0 { + return definitionFailure(err, diagnostics) + } + operationErrors = append(operationErrors, err) + break + } + + seed := smokeRequestForKind(input.request, kind) + acquired, operationErr := suite.SmokeTest(invocation.Context, SmokeRequest{Selection: input.selection, Request: seed}, input.selected, state) + if acquired.Value == nil && operationErr == nil { + operationErr = errors.New("PromptKit smoke test returned neither a report nor an error") + } + if err := promptKitContextError(invocation.Context); err != nil && !errors.Is(operationErr, err) { + if operationErr == nil { + operationErr = err + } else { + operationErr = errors.Join(operationErr, err) + } + } + + results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr}) + diagnostics = append(diagnostics, acquired.Diagnostics...) + if operationErr != nil { + operationErrors = append(operationErrors, operationErr) + } + if err := promptKitContextError(invocation.Context); err != nil { + break + } + } + + if len(results) == 0 { + return definitionFailure(errors.Join(operationErrors...), diagnostics) + } + output, encodeErr := encodeSmokeResults(results) + if encodeErr != nil { + return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false) + } + return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...)) +} + +func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest { + request := seed + request.Kind = kind + if seed.Execution != nil { + execution := *seed.Execution + execution.Temperature = copyFloat64(seed.Execution.Temperature) + execution.MaxTokens = copyInt(seed.Execution.MaxTokens) + execution.TopP = copyFloat64(seed.Execution.TopP) + execution.TimeoutSeconds = copyInt(seed.Execution.TimeoutSeconds) + execution.ReasoningEffort = copyString(seed.Execution.ReasoningEffort) + request.Execution = &execution + } + request.TransientRetries = copyInt(seed.TransientRetries) + return request +} + +func copyFloat64(value *float64) *float64 { + if value == nil { + return nil + } + copy := *value + return © +} + +func copyInt(value *int) *int { + if value == nil { + return nil + } + copy := *value + return © +} + +func copyString(value *string) *string { + if value == nil { + return nil + } + copy := *value + return © +} + func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome { written, deliveryErr := invocation.Streams.Out.Write(output) if deliveryErr == nil && written != len(output) { diff --git a/integrations/ckprompt/smoke_command_test.go b/integrations/ckprompt/smoke_command_test.go index e30b29f..978b960 100644 --- a/integrations/ckprompt/smoke_command_test.go +++ b/integrations/ckprompt/smoke_command_test.go @@ -93,8 +93,13 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) { providerCalls = 0 output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeUsage || providerCalls != 0 || output.Len() != 0 { - t.Fatalf("unexpanded aggregate token = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) + var aggregate struct { + Tests []struct { + Kind string `yaml:"kind"` + } `yaml:"tests"` + } + if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 4 || yaml.Unmarshal(output.Bytes(), &aggregate) != nil || len(aggregate.Tests) != 4 { + t.Fatalf("aggregate smoke result = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) } }