diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index c156ba8..0cf2705 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Dependency upgrades, discovery, and smoke testing implementation plan -Status: Stages 1–8 complete; Stage 9 is next. +Status: Stages 1–9 complete; Stage 10 is next. ## Authority and baseline @@ -492,6 +492,19 @@ No live provider or real credentials are needed. **Complete when:** one concrete smoke request reaches the native API through an independently optional, composable adapter. +**Completed:** 2026-10-17. Added `SmokeRequest`, the independently optional +`Suite.SmokeTest` callback, and `WithSmokeTesting`. The helper preserves other +suite capabilities and issues, invokes an optional builder before loading, +forwards its complete request unchanged, retains loader diagnostics, checks +cancellation before acquisition, and returns the native report and error +without filtering either. Focused tests cover nil-builder forwarding, +authoritative builder output, context/input/state, builder failure before load, +nil-loader issue retention, cancellation, loader diagnostics, and native error +identity. PromptKit's injected-client fixture returned a report with nil error +when the client returned an error, so public fixture evidence for a +report-plus-error pair is unavailable; the adapter returns both native values +directly. Focused adapter tests and `GOTOOLCHAIN=local go build ./...` passed. + ## Stage 10: Implement faithful smoke YAML projection **Primary files:** private `smoke_report.go` and focused projection tests. diff --git a/integrations/ckprompt/adapters.go b/integrations/ckprompt/adapters.go index 8c37735..9f4256c 100644 --- a/integrations/ckprompt/adapters.go +++ b/integrations/ckprompt/adapters.go @@ -63,8 +63,9 @@ type RequirementsRequest struct { } // Suite describes application-owned input, shared state, and optional catalog, -// named inspection, requirements discovery, and concrete request validation -// capabilities. Its private issues retain adapter construction failures. +// named inspection, requirements discovery, concrete request validation, and +// smoke testing capabilities. Its private issues retain adapter construction +// failures. // Factory results and callback effects remain caller-owned: Command does not // deep-copy arbitrary objects or repair aliases returned by factories. // A reached operation prepares one input, applies its captured selectors, and @@ -102,6 +103,11 @@ type Suite[I, S any] struct { // capabilities. The acquisition carries diagnostics and an empty value on // success; PromptKit errors are returned unchanged where supported. ValidateRequest func(context.Context, ValidationRequest, I, S) (commandkit.Acquired[struct{}], error) + // SmokeTest runs one concrete native smoke-test operation after input and + // state preparation. It is optional; nil omits only the smoke-test command. + // The acquisition retains a native report even when the callback also + // returns an error. + SmokeTest func(context.Context, SmokeRequest, I, S) (commandkit.Acquired[*promptkit.SmokeTestReport], error) // ListPrompts lists prompt identities after input and state preparation. // It is optional independently of the other catalogs; nil omits only this // catalog command. diff --git a/integrations/ckprompt/adapters_test.go b/integrations/ckprompt/adapters_test.go index 73e060b..a1a4795 100644 --- a/integrations/ckprompt/adapters_test.go +++ b/integrations/ckprompt/adapters_test.go @@ -91,7 +91,7 @@ func adapterEngine(t *testing.T) *promptkit.Engine { return adapterEngineFrom(t, "adapter fixture", adapterPrompt) } -func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Engine { +func adapterEngineFrom(t *testing.T, label, configuration string, options ...promptkit.Option) *promptkit.Engine { t.Helper() inputs, err := promptkit.FilesystemInputs(label, fstest.MapFS{ "config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)}, @@ -101,13 +101,159 @@ func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Eng } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}}, - }}) + }}, options...) if err != nil { t.Fatalf("construct PromptKit fixture: %v", err) } return engine } +type recordingSmokeClient struct { + request promptkit.GenerateRequest + response *promptkit.GenerateResponse + calls int +} + +func (client *recordingSmokeClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.calls++ + client.request = request + return client.response, nil +} + +func TestWithSmokeTestingForwardsAuthoritativeNativeRequestAndRetainsReport(t *testing.T) { + client := &recordingSmokeClient{ + response: &promptkit.GenerateResponse{Content: "fixture response"}, + } + engine := adapterEngineFrom(t, "smoke adapter fixture", adapterPrompt, promptkit.WithLLMClient(client)) + type input struct{ label string } + type state struct{ label string } + type contextKey struct{} + wantContext := context.WithValue(context.Background(), contextKey{}, "smoke invocation") + wantInput := input{label: "typed input"} + wantState := state{label: "shared state"} + wantSelection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}} + loaderCalls, builderCalls := 0, 0 + seed := promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "unused", APIKey: "seed credential"} + built := promptkit.SmokeTestRequest{ + Kind: promptkit.SmokeTestTextInference, + PromptID: "briefing", + APIKey: "builder credential", + Execution: &promptkit.ExecutionTargetOverride{Model: "builder-model"}, + } + suite := WithSmokeTesting(Suite[input, state]{}, + func(ctx context.Context, selection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) { + loaderCalls++ + if ctx.Value(contextKey{}) != "smoke invocation" || selection != wantSelection || gotInput != wantInput || gotState != wantState { + t.Errorf("loader args = ctx=%v selection=%#v input=%#v state=%#v", ctx.Value(contextKey{}), selection, gotInput, gotState) + } + return engine, nil + }, + func(ctx context.Context, request SmokeRequest, gotInput input, gotState state) (promptkit.SmokeTestRequest, error) { + builderCalls++ + if ctx.Value(contextKey{}) != "smoke invocation" || request.Selection != wantSelection || request.Request.Kind != promptkit.SmokeTestAuthentication || gotInput != wantInput || gotState != wantState { + t.Errorf("builder args = ctx=%v request=%#v input=%#v state=%#v", ctx.Value(contextKey{}), request, gotInput, gotState) + } + return built, nil + }, + ) + if suite.SmokeTest == nil { + t.Fatal("WithSmokeTesting did not install a callback") + } + + acquired, err := suite.SmokeTest(wantContext, SmokeRequest{Selection: wantSelection, Request: seed}, wantInput, wantState) + if err != nil || acquired.Value == nil || len(acquired.Diagnostics) != 0 || loaderCalls != 1 || builderCalls != 1 { + t.Fatalf("smoke acquisition = (%#v, %v), loaders/builders=%d/%d", acquired, err, loaderCalls, builderCalls) + } + if acquired.Value.Kind != promptkit.SmokeTestTextInference || client.calls != 1 || client.request.Target.Model != "builder-model" || client.request.APIKey != "builder credential" { + t.Fatalf("native smoke request/report = request %#v report %#v calls=%d", client.request, acquired.Value, client.calls) + } +} + +func TestWithSmokeTestingNilBuilderForwardsSeedAndBuilderFailureSkipsLoader(t *testing.T) { + client := &recordingSmokeClient{response: &promptkit.GenerateResponse{Content: "fixture response"}} + engine := adapterEngineFrom(t, "smoke adapter fixture", adapterPrompt, promptkit.WithLLMClient(client)) + seed := promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, PromptID: "briefing", APIKey: "seed credential"} + loaderCalls := 0 + suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loaderCalls++ + return engine, nil + }, nil) + acquired, err := suite.SmokeTest(context.Background(), SmokeRequest{Request: seed}, struct{}{}, struct{}{}) + if err != nil || acquired.Value == nil || client.calls != 1 || client.request.APIKey != "seed credential" || len(client.request.Prompt.Messages) != 1 { + t.Fatalf("nil-builder smoke request = (%#v, %v), client=%#v", acquired, err, client) + } + acquired, err = suite.SmokeTest(context.Background(), SmokeRequest{Request: promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestKind("unknown")}}, struct{}{}, struct{}{}) + if !errors.Is(err, promptkit.ErrInvalidRequest) || acquired.Value != nil { + t.Fatalf("native smoke error = (%#v, %v), want invalid request", acquired, err) + } + + buildErr := errors.New("request builder failed") + loaderCalls = 0 + suite = WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loaderCalls++ + return engine, nil + }, func(context.Context, SmokeRequest, struct{}, struct{}) (promptkit.SmokeTestRequest, error) { + return promptkit.SmokeTestRequest{}, buildErr + }) + acquired, err = suite.SmokeTest(context.Background(), SmokeRequest{}, struct{}{}, struct{}{}) + if err != buildErr || acquired.Value != nil || loaderCalls != 0 { + t.Fatalf("builder failure acquisition = (%#v, %v), loader calls=%d", acquired, err, loaderCalls) + } +} + +func TestWithSmokeTestingRetainsNilLoaderIssueAndChecksCancellationBeforeBuild(t *testing.T) { + sentinel := errors.New("existing suite issue") + suite := WithSmokeTesting(Suite[int, int]{issues: []error{sentinel}}, nil, nil) + if suite.SmokeTest != nil || len(suite.issues) != 2 || suite.issues[0] != sentinel || !strings.Contains(suite.issues[1].Error(), "engine loader is nil") { + t.Fatalf("nil-loader smoke wiring = %#v", suite) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + buildCalls, loadCalls := 0, 0 + suite = WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loadCalls++ + return nil, nil + }, func(context.Context, SmokeRequest, int, int) (promptkit.SmokeTestRequest, error) { + buildCalls++ + return promptkit.SmokeTestRequest{}, nil + }) + acquired, err := suite.SmokeTest(ctx, SmokeRequest{}, 0, 0) + if !errors.Is(err, context.Canceled) || acquired.Value != nil || buildCalls != 0 || loadCalls != 0 { + t.Fatalf("pre-canceled smoke = (%#v, %v), builders/loaders=%d/%d", acquired, err, buildCalls, loadCalls) + } + + ctx, cancel = context.WithCancel(context.Background()) + buildCalls, loadCalls = 0, 0 + suite = WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + loadCalls++ + return nil, nil + }, func(context.Context, SmokeRequest, int, int) (promptkit.SmokeTestRequest, error) { + buildCalls++ + cancel() + return promptkit.SmokeTestRequest{}, nil + }) + acquired, err = suite.SmokeTest(ctx, SmokeRequest{}, 0, 0) + if !errors.Is(err, context.Canceled) || acquired.Value != nil || buildCalls != 1 || loadCalls != 0 { + t.Fatalf("canceled-after-build smoke = (%#v, %v), builders/loaders=%d/%d", acquired, err, buildCalls, loadCalls) + } +} + +func TestWithSmokeTestingRetainsLoaderDiagnosticsAndErrorIdentity(t *testing.T) { + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid smoke configuration", []byte("broken: ["))) + if loadErr == nil { + t.Fatal("invalid configuration unexpectedly loaded") + } + suite := WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { + return nil, loadErr + }, nil) + acquired, err := suite.SmokeTest(context.Background(), SmokeRequest{}, 0, 0) + var retained *configkit.LoadError + if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || acquired.Value != nil || len(acquired.Diagnostics) == 0 { + t.Fatalf("smoke load failure = (%#v, %v)", acquired, err) + } +} + func TestConfigScopeUsesNativeInspectionFacetsAndInvocationValues(t *testing.T) { type input struct{ token string } type state struct{ token string } diff --git a/integrations/ckprompt/smoke.go b/integrations/ckprompt/smoke.go new file mode 100644 index 0000000..23da56c --- /dev/null +++ b/integrations/ckprompt/smoke.go @@ -0,0 +1,69 @@ +package ckprompt + +import ( + "context" + "errors" + "fmt" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" +) + +// SmokeRequest carries the configuration selection and seed request for one +// concrete PromptKit smoke-test operation. +type SmokeRequest struct { + // Selection identifies the configuration sources used to load the engine. + Selection ckconfig.Selection + // Request contains the native smoke-test operation and its selected inputs. + Request promptkit.SmokeTestRequest +} + +// WithSmokeTesting attaches PromptKit-backed smoke testing to suite while +// preserving its other capabilities and retained issues. The optional builder +// runs before engine loading and receives the seed request, input, state, and +// invocation context. Its complete returned request is passed to PromptKit; +// without a builder, the seed is forwarded unchanged. Reports and errors from +// PromptKit are both retained. A nil loader is kept as a suite wiring issue and +// does not install a smoke-test callback. +func WithSmokeTesting[I, S any]( + suite Suite[I, S], + load func(context.Context, ckconfig.Selection, I, S) (*promptkit.Engine, error), + build func(context.Context, SmokeRequest, I, S) (promptkit.SmokeTestRequest, error), +) Suite[I, S] { + if load == nil { + issues := append([]error(nil), suite.issues...) + suite.issues = append(issues, errors.New("PromptKit engine loader is nil")) + return suite + } + + suite.SmokeTest = func(ctx context.Context, request SmokeRequest, input I, state S) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[*promptkit.SmokeTestReport]{}, fmt.Errorf("PromptKit smoke test before building: %w", err) + } + + concreteRequest := request.Request + if build != nil { + built, err := build(ctx, request, input, state) + if err != nil { + return commandkit.Acquired[*promptkit.SmokeTestReport]{}, err + } + concreteRequest = built + } + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[*promptkit.SmokeTestReport]{}, fmt.Errorf("PromptKit smoke test after building: %w", err) + } + + engine, _, diagnostics, err := loadPromptKitEngine(ctx, request.Selection, input, state, "", load) + if err != nil { + return commandkit.Acquired[*promptkit.SmokeTestReport]{Diagnostics: diagnostics}, err + } + if err := promptKitContextError(ctx); err != nil { + return commandkit.Acquired[*promptkit.SmokeTestReport]{Diagnostics: diagnostics}, fmt.Errorf("PromptKit smoke test before native operation: %w", err) + } + + report, err := engine.SmokeTest(ctx, concreteRequest) + return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: report, Diagnostics: diagnostics}, err + } + return suite +}