From 80f31d2ec0afa7d035c33838b8fad1d6b96e511f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 17 Oct 2026 19:00:03 +0000 Subject: [PATCH] Verify PromptKit smoke command lifecycles --- docs/roadmap/implementation.md | 10 +- integrations/ckprompt/smoke_command_test.go | 10 + integrations/ckprompt/smoke_lifecycle_test.go | 266 ++++++++++++++++++ 3 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 integrations/ckprompt/smoke_lifecycle_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ef72fb8..f318b67 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–12 complete; Stage 13 is next. +Status: Stages 1–13 complete; Stage 14 is next. ## Authority and baseline @@ -659,6 +659,14 @@ Assert callback requests and invocation state, not native target resolution. **Complete when:** the new command obeys selected grammar, presence, capture, availability, and lifetime contracts. +**Completed:** 2026-10-17. Added smoke lifecycle coverage for repeated scalar +values, explicit zero and empty overrides, config/overlay presence, target +combinations, and reset across invocations. Usage and input-check failures +stop before state/provider work. Availability, captured callbacks/selectors, +retained installation issues, and help/version inactivity are verified, along +with nil-factory zero values and concrete kinds reaching builders. Focused smoke +tests and the full ckprompt suite passed with `GOTOOLCHAIN=local`. + ## Stage 14: Verify smoke outcomes and report delivery **Primary files:** focused smoke outcome tests and existing writer/diagnostic diff --git a/integrations/ckprompt/smoke_command_test.go b/integrations/ckprompt/smoke_command_test.go index 978b960..0ae0110 100644 --- a/integrations/ckprompt/smoke_command_test.go +++ b/integrations/ckprompt/smoke_command_test.go @@ -126,6 +126,16 @@ func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) { if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 { t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String()) } + + app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + return commandkit.Acquired[*promptkit.SmokeTestReport]{}, promptkit.ErrInvalidRequest + }, + })) + outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrInvalidRequest) { + t.Fatalf("native semantic error classification = %#v", outcome) + } } type smokeShortWriter struct{} diff --git a/integrations/ckprompt/smoke_lifecycle_test.go b/integrations/ckprompt/smoke_lifecycle_test.go new file mode 100644 index 0000000..b0dcf69 --- /dev/null +++ b/integrations/ckprompt/smoke_lifecycle_test.go @@ -0,0 +1,266 @@ +package ckprompt + +import ( + "bytes" + "context" + "errors" + "io" + "reflect" + "strings" + "testing" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" +) + +func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T) { + type input struct{ campaign string } + type received struct { + selection ckconfig.Selection + request promptkit.SmokeTestRequest + input input + } + campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString, + func(target *input, value commandkit.Supplied[string]) { target.campaign = value.Value }) + var requests []received + suite := Suite[input, struct{}]{ + NewInput: func() input { return input{} }, + Selectors: []commandkit.Flag[input]{campaignFlag}, + SmokeTest: func(_ context.Context, request SmokeRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + requests = append(requests, received{selection: request.Selection, request: request.Request, input: value}) + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil + }, + } + app := newPromptTestApp(t, Command(suite)) + + first := app.Execute(context.Background(), []string{ + "promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml", + "--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final", + "--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt", + "--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model", + "--temperature=0", "--transient-retries=0", + }, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 { + t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests) + } + wantSelection := ckconfig.Selection{ + Config: commandkit.Supplied[string]{Value: "final.yml", Set: true}, + Overlay: commandkit.Supplied[string]{Value: "final-overlay.yml", Set: true}, + } + got := requests[0] + if got.selection != wantSelection || got.input.campaign != "final" || got.request.BackendID != "final-backend" || got.request.PromptID != "final-prompt" || got.request.ProfileID != "final-profile" { + t.Fatalf("last supplied values = %#v", got) + } + if got.request.Execution == nil || got.request.Execution.Model != "final-model" || got.request.Execution.Temperature == nil || *got.request.Execution.Temperature != 0 || got.request.TransientRetries == nil || *got.request.TransientRetries != 0 { + t.Fatalf("last execution values = %#v", got.request) + } + + second := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if second.Kind != commandkit.OutcomeSuccess || len(requests) != 2 { + t.Fatalf("second smoke invocation = %#v, requests=%#v", second, requests) + } + got = requests[1] + if got.selection.Config.Set || got.selection.Overlay.Set || got.input.campaign != "" || got.request.BackendID != "" || got.request.PromptID != "" || got.request.ProfileID != "" || got.request.Execution != nil || got.request.TransientRetries != nil { + t.Fatalf("values leaked between invocations: %#v", got) + } +} + +func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) { + for _, test := range []struct { + name string + args []string + }{ + {name: "missing kind", args: []string{"promptkit", "test"}}, + {name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}}, + {name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}}, + {name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}}, + {name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}}, + {name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}}, + {name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}}, + {name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=json"}}, + {name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}}, + } { + t.Run(test.name, func(t *testing.T) { + newInputCalls, checkCalls, newStateCalls, providerCalls := 0, 0, 0, 0 + suite := Suite[int, struct{}]{ + NewInput: func() int { newInputCalls++; return 1 }, + NewState: func() struct{} { newStateCalls++; return struct{}{} }, + CheckInput: func(ckconfig.Selection, int) error { + checkCalls++ + return nil + }, + SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + providerCalls++ + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil + }, + } + outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), test.args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || checkCalls != 0 || newStateCalls != 0 || providerCalls != 0 { + t.Fatalf("preflight failure = %#v, input/check/state/provider=%d/%d/%d/%d", outcome, newInputCalls, checkCalls, newStateCalls, providerCalls) + } + if newInputCalls != 0 { + t.Fatalf("preflight failure invoked suite input factory %d times", newInputCalls) + } + }) + } + + inputErr := errors.New("input rejected") + newStateCalls, providerCalls := 0, 0 + suite := Suite[int, struct{}]{ + NewInput: func() int { return 1 }, + NewState: func() struct{} { newStateCalls++; return struct{}{} }, + CheckInput: func(ckconfig.Selection, int) error { + return inputErr + }, + SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + providerCalls++ + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil + }, + } + outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || !errors.Is(outcome.Cause, inputErr) || newStateCalls != 0 || providerCalls != 0 { + t.Fatalf("input-check failure reached state/provider: %#v state=%d provider=%d", outcome, newStateCalls, providerCalls) + } +} + +func TestSmokeTestCommandAvailabilityCaptureAndHelpLifecycle(t *testing.T) { + type input struct{ campaign string } + var originalCalls, replacementCalls, inputCalls, stateCalls int + var capturedInput input + campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString, + func(value *input, supplied commandkit.Supplied[string]) { value.campaign = supplied.Value }) + replacementFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "season"}, (*string)(nil), parseDefinitionString, + func(value *input, supplied commandkit.Supplied[string]) { + value.campaign = "replacement:" + supplied.Value + }) + selectors := []commandkit.Flag[input]{campaignFlag} + suite := Suite[input, int]{ + NewInput: func() input { inputCalls++; return input{} }, + NewState: func() int { stateCalls++; return 1 }, + Selectors: selectors, + SmokeTest: func(_ context.Context, _ SmokeRequest, value input, _ int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + originalCalls++ + capturedInput = value + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil + }, + } + declaration := Command(suite) + suite.SmokeTest = func(context.Context, SmokeRequest, input, int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + replacementCalls++ + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil + } + selectors[0] = replacementFlag + app := newPromptTestApp(t, declaration) + var help bytes.Buffer + outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 { + t.Fatalf("smoke help lifecycle = (%#v, %q), input/state/providers=%d/%d/%d/%d", outcome, help.String(), inputCalls, stateCalls, originalCalls, replacementCalls) + } + outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 { + t.Fatalf("version lifecycle = %#v, input/state/providers=%d/%d/%d/%d", outcome, inputCalls, stateCalls, originalCalls, replacementCalls) + } + + outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--campaign=spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || originalCalls != 1 || replacementCalls != 0 || inputCalls != 1 || stateCalls != 1 || capturedInput.campaign != "spring" { + t.Fatalf("captured callback/selector = (%#v, %#v), calls=%d/%d input/state=%d/%d", outcome, capturedInput, originalCalls, replacementCalls, inputCalls, stateCalls) + } + + otherApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ + ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { + return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil + }, + })) + help.Reset() + outcome = otherApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(help.String(), "\n test ") { + t.Fatalf("smoke command installed without callback: (%#v, %q)", outcome, help.String()) + } + + issueSuite := WithSmokeTesting(Suite[struct{}, struct{}]{ + InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) { + return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil + }, + }, nil, nil) + issueSuite.SmokeTest = nil + if _, err := commandkit.New(commandkit.Identity{Name: "example"}, Command(issueSuite)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") { + t.Fatalf("narrowed suite lost smoke wiring issue: %v", err) + } +} + +func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) { + wantKinds := []promptkit.SmokeTestKind{ + promptkit.SmokeTestReachability, + promptkit.SmokeTestAuthentication, + promptkit.SmokeTestTextInference, + promptkit.SmokeTestStructuredOutput, + } + var builtKinds []promptkit.SmokeTestKind + loadErr := errors.New("engine unavailable") + loads := 0 + suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loads++ + return nil, loadErr + }, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) { + builtKinds = append(builtKinds, request.Request.Kind) + return request.Request, nil + }) + app := newPromptTestApp(t, Command(suite)) + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || loads != 4 || !reflect.DeepEqual(builtKinds, wantKinds) { + t.Fatalf("aggregate builder lifecycle = %#v, built=%#v loads=%d", outcome, builtKinds, loads) + } +} + +func TestSmokeTestHelpAndVersionLeaveWrappedProviderIdle(t *testing.T) { + newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls := 0, 0, 0, 0, 0 + suite := WithSmokeTesting(Suite[int, int]{ + NewInput: func() int { newInputCalls++; return 1 }, + NewState: func() int { newStateCalls++; return 2 }, + CheckInput: func(ckconfig.Selection, int) error { + checkCalls++ + return nil + }, + }, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loadCalls++ + return nil, errors.New("loader should be idle") + }, func(_ context.Context, request SmokeRequest, _ int, _ int) (promptkit.SmokeTestRequest, error) { + buildCalls++ + return request.Request, nil + }) + app := newPromptTestApp(t, Command(suite)) + for _, args := range [][]string{{"help", "promptkit"}, {"version"}} { + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("%v outcome = %#v", args, outcome) + } + } + if newInputCalls != 0 || checkCalls != 0 || newStateCalls != 0 || buildCalls != 0 || loadCalls != 0 { + t.Fatalf("help/version invoked smoke work: input/check/state/build/load=%d/%d/%d/%d/%d", newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls) + } +} + +func TestSmokeTestNilFactoriesSupplyZeroValuesForAggregate(t *testing.T) { + checkCalls, providerCalls := 0, 0 + suite := Suite[struct{ value string }, struct{ count int }]{ + CheckInput: func(_ ckconfig.Selection, value struct{ value string }) error { + checkCalls++ + if value.value != "" { + t.Errorf("nil input factory value = %#v", value) + } + return nil + }, + SmokeTest: func(_ context.Context, _ SmokeRequest, value struct{ value string }, status struct{ count int }) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + providerCalls++ + if value.value != "" || status.count != 0 { + t.Errorf("nil factories yielded input/state (%#v, %#v)", value, status) + } + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil + }, + } + outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || checkCalls != 1 || providerCalls != 4 { + t.Fatalf("nil-factory aggregate = %#v, checks/providers=%d/%d", outcome, checkCalls, providerCalls) + } +}