From 114c7da7becc26b718e45af858420dac612a6e54 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 18 Oct 2026 13:30:02 +0000 Subject: [PATCH] Keep aggregate smoke tests focused on behavior --- docs/roadmap/remediation.md | 12 ++++++++++-- integrations/ckprompt/smoke_aggregate_test.go | 13 ------------- integrations/ckprompt/smoke_lifecycle_test.go | 6 ++---- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 907668b..1b79bdc 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -1,6 +1,6 @@ # Dependency upgrade and PromptKit smoke review remediation -Status: in progress; R-1 and R-2 complete; R-3 and R-4 remain open. +Status: in progress; R-1 through R-3 complete; R-4 remains open. ## Review scope and evidence @@ -127,7 +127,7 @@ Production code is unchanged. ## R-3: Remove incidental loader-count and pointer-identity contracts -Status: open. Priority: P3. Category: unnecessary implementation constraints. +Status: complete. Priority: P3. Category: unnecessary implementation constraints. **Locations:** @@ -167,6 +167,14 @@ pass. The tests continue to detect cross-test value contamination and missing operations, while accepting equivalent acquisition and storage implementations. No production-code change is required. +**Completed:** 2026-10-18. Removed the aggregate loader counter and pairwise +pointer-address assertions/bookkeeping. The tests retain ordered concrete +builder/callback inputs, shared input/state, mutation-isolated seed values, +continuation, cancellation, reports, and error identity. Focused aggregate, +builder, help/version, and builder-failure tests and the complete ckprompt suite, +including the composed consumer, passed with Go 1.26.8 and offline dependency +resolution. Production code is unchanged. + ## R-4: Reconcile the feature roadmap with completed implementation Status: open. Priority: P3. Category: stale implementation status. diff --git a/integrations/ckprompt/smoke_aggregate_test.go b/integrations/ckprompt/smoke_aggregate_test.go index 23e0c96..d260d4f 100644 --- a/integrations/ckprompt/smoke_aggregate_test.go +++ b/integrations/ckprompt/smoke_aggregate_test.go @@ -28,9 +28,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) nativeErr := errors.New("authentication smoke failed") newInputCalls, checkCalls, newStateCalls := 0, 0, 0 var gotKinds []promptkit.SmokeTestKind - var executionPointers []*promptkit.ExecutionTargetOverride - var temperaturePointers []*float64 - var retryPointers []*int var firstInput *input var sharedState *state suite := Suite[*input, *state]{ @@ -60,9 +57,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 { t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request) } - executionPointers = append(executionPointers, request.Request.Execution) - temperaturePointers = append(temperaturePointers, request.Request.Execution.Temperature) - retryPointers = append(retryPointers, request.Request.TransientRetries) if len(gotKinds) == 1 { request.Request.Execution.Model = "builder mutation" *request.Request.Execution.Temperature = 0.9 @@ -86,13 +80,6 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) { t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds) } - for index := range executionPointers { - for other := index + 1; other < len(executionPointers); other++ { - if executionPointers[index] == executionPointers[other] || temperaturePointers[index] == temperaturePointers[other] || retryPointers[index] == retryPointers[other] { - t.Fatalf("request pointers were reused between operations %d and %d", index, other) - } - } - } var document struct { Tests []struct { diff --git a/integrations/ckprompt/smoke_lifecycle_test.go b/integrations/ckprompt/smoke_lifecycle_test.go index b0dcf69..bf22fe6 100644 --- a/integrations/ckprompt/smoke_lifecycle_test.go +++ b/integrations/ckprompt/smoke_lifecycle_test.go @@ -198,9 +198,7 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) { } var builtKinds []promptkit.SmokeTestKind loadErr := errors.New("engine unavailable") - loads := 0 suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { - loads++ return nil, loadErr }, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) { builtKinds = append(builtKinds, request.Request.Kind) @@ -208,8 +206,8 @@ func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) { }) app := newPromptTestApp(t, Command(suite)) outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || loads != 4 || !reflect.DeepEqual(builtKinds, wantKinds) { - t.Fatalf("aggregate builder lifecycle = %#v, built=%#v loads=%d", outcome, builtKinds, loads) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || !reflect.DeepEqual(builtKinds, wantKinds) { + t.Fatalf("aggregate builder lifecycle = %#v, built=%#v", outcome, builtKinds) } }