From 1289868ed07bab028bb427c9431d1f63582ddadd Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 7 Nov 2026 05:01:30 +0000 Subject: [PATCH] Update the composed PromptKit consumer --- docs/roadmap/implementation.md | 6 +- .../ckprompt/examples/composed/README.md | 42 +++-- .../ckprompt/examples/composed/main.go | 49 +++-- .../ckprompt/examples/composed/main_test.go | 169 +++++++++++++++++- 4 files changed, 236 insertions(+), 30 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 29df0b7..145235a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Shared reporting and PromptKit v0.20.0 implementation plan -Status: implementation in progress; Stages 1–21 complete. +Status: implementation in progress; Stages 1–22 complete. ## Scope, authority, and execution @@ -784,6 +784,10 @@ do not initialize application runtime state. ## Stage 22: Composed consumer and application-owned flag reuse +**Complete.** The composed consumer exercises request preview, namespaces, +reusable flags, independent overrides, and failure evidence with invocation-local +caching. The root, ckconfig, and ckprompt workspace suites passed with Go 1.27.0. + Update the maintained [composed consumer](../../integrations/ckprompt/examples/composed) to install `ckprompt.Declarations`, explicit empty native namespace defaults (enabling root defaults plus opt-in selectors), request preparation, extended diff --git a/integrations/ckprompt/examples/composed/README.md b/integrations/ckprompt/examples/composed/README.md index 5133418..f8ce759 100644 --- a/integrations/ckprompt/examples/composed/README.md +++ b/integrations/ckprompt/examples/composed/README.md @@ -1,7 +1,9 @@ # Composed consumer This example is part of the ckprompt module and uses core v0.3.0, ckconfig -v0.3.0, ConfigKit v1.1.0, and PromptKit v0.18.0. It requires Go 1.27.0. +v0.3.0, ConfigKit v1.1.0, and PromptKit v0.20.0. It requires Go 1.27.0. +The current work uses the aligned local workspace until the required core and +ckconfig releases are published. This example composes application configuration, PromptKit configuration, PromptKit comparison and catalogs, definition and requirements inspection, and @@ -12,6 +14,7 @@ go run . config scopes go run . config validate --scope all --campaign spring --limit=0 go run . config diff --scope app --config fixtures/before.yml --against-config fixtures/after.yml go run . config diff --scope app --config fixtures/before.yml --against-config fixtures/after.yml --campaign spring --limit=0 +go run . config diff --scope app --campaign spring --limit=0 --against-campaign autumn --against-limit=0 --set '/campaigns/spring/model="left"' --against-set '/campaigns/spring/model="right"' --details go run . config diff --scope promptkit --config fixtures/before.yml --against-config fixtures/after.yml --campaign spring --against-campaign spring go run . promptkit list prompts --config fixtures/before.yml --campaign spring go run . promptkit list profiles --config fixtures/after.yml --campaign autumn --format yaml @@ -22,6 +25,8 @@ go run . list collections --campaign spring go run . pk list bundles --campaign spring go run . promptkit list prompts --namespace field-notes --campaign spring go run . promptkit show prompt briefing --campaign spring --format yaml +go run . promptkit show prompt briefing --campaign spring --prompt-namespace field-notes +go run . promptkit show prepared briefing --campaign spring --input topic=sprint --profile spring --extra-params '{"campaign":"preview"}' go run . pk show prompt briefing --campaign spring --fmt yaml go run . promptkit show profile spring --campaign spring go run . promptkit requirements show briefing --campaign spring @@ -34,6 +39,7 @@ go run . smoke text_inference --campaign spring --llm-prompt briefing --llm-prof go run . promptkit test all --campaign spring --llm-prompt briefing --llm-profile spring go run . pk --hlep go run . session restore session-42 --campaign spring +go run . request inspect --llm-prompt briefing --llm-profile= --llm-model example --llm-input-text topic=sprint --format json ``` `newApp` installs application-owned command and flag aliases alongside the @@ -51,8 +57,11 @@ Bash, then load a shell script as described in the in `composed-example config show --config=fixtures/`. The already-entered `pk` alias selects the PromptKit command context without loading an engine. -The composition calls `ckconfig.Command(suite)` and `ckprompt.Command(definitions)` -directly in a `[]commandkit.Declaration` with root-installed routing rules. +The composition installs `ckconfig.Command(suite)` and the complete +`ckprompt.Declarations(definitions)` in a `[]commandkit.Declaration` with +root-installed routing rules. The PromptKit declarations include scoped aliases; +the application-owned `request inspect` command reuses the flag collections +with their canonical names and does not install those aliases. Group children remain canonical `Command` values. For command-only slices, convert each `Command` to `Declaration` before appending aliases. The [consumer guide](../../../../docs/consumers/guide.md#application-owned-aliases) @@ -79,10 +88,13 @@ Prompt inspection uses an ID only. The default PromptKit bundle also contributes profiles and a backend to the engine inventory and source reports. The synthetic `field-notes` collection provides one backend, profile, and prompt for collection discovery and native namespace filtering. The example's shared run cache keys successful results by -the complete supplied configuration/overlay selection and the supplied -campaign, including presence. The campaign override is applied through ConfigKit -before loading. These are choices for this example's loader and do not define a -CommandKit cache policy. +the complete supplied configuration/overlay selection, CLI overrides, and the +supplied campaign, including presence. The campaign override and application +scope's `--set` records are applied through ConfigKit before loading. The +PromptKit configuration scope does not enable `--set` because its released +configuration loader has no override handoff. An aggregate operation therefore +rejects overrides when that scope is selected. These are choices for this +example's loaders and do not define a CommandKit cache policy. Smoke testing is explicitly wired with `ckprompt.WithSmokeTesting` and its optional request builder is nil, so CommandKit forwards the selected native @@ -114,17 +126,19 @@ not execute the prompt or establish input, credential, or execution readiness. See the [CLI reference](../../../../docs/cli.md#promptkit-requirements) for the command and report contract. -The validation commands exercise concrete PromptKit request preparation. +The validation and preview commands exercise concrete PromptKit request preparation. The first uses the builder's campaign-specific default topic variable and the operator's topic file. The second also selects the notes branch and supplies its -file. Both succeed and report only `prompt_id` and `valid: true`; they do not -execute the prompt. The final command intentionally fails because the prompt's +file. Validation reports only `prompt_id` and `valid: true`; preview returns the +prepared request, including rendered messages and effective provider settings. +Neither executes the prompt. The validation command without inputs intentionally +fails because the prompt's required `topic` artifact is not supplied. The builder defaults variables from the selected campaign but leaves artifact bindings to the operator. The `requirements show` commands remain discovery reports: partial discovery can succeed even when a concrete request would fail validation. -Run `GOWORK=off go test ./...` from this example directory to check it against -the module's declared dependencies. A local workspace can instead select the -three current sources, as described in the -[developer guide](../../../../docs/development.md#documentation-validation). +Use the aligned local workspace to test the three current sources, as described in +the [developer guide](../../../../docs/development.md#documentation-validation). +Once released dependency requirements are aligned, `GOWORK=off go test ./...` +from this example directory checks it against its declared dependencies. diff --git a/integrations/ckprompt/examples/composed/main.go b/integrations/ckprompt/examples/composed/main.go index fb3b1b8..3a7692c 100644 --- a/integrations/ckprompt/examples/composed/main.go +++ b/integrations/ckprompt/examples/composed/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "reflect" "strconv" "go.maximumdirect.net/ckconfig" @@ -31,16 +32,16 @@ type inspectionInput struct { Limit commandkit.Supplied[int] } -type applicationCacheKey struct { - config commandkit.Supplied[string] - overlay commandkit.Supplied[string] - campaign commandkit.Supplied[string] +type applicationCacheEntry struct { + selection ckconfig.Selection + campaign commandkit.Supplied[string] + result *configkit.Result[appConfiguration] } // runCache belongs to one invocation and is shared only where the application // chooses to pass it to its loaders. type runCache struct { - applications map[applicationCacheKey]*configkit.Result[appConfiguration] + applications []applicationCacheEntry llmClient promptkit.LLMClient } @@ -60,6 +61,12 @@ type restoreInput struct { Session string } +type requestInputs struct { + Execution ckprompt.ExecutionInput `json:"execution"` + Resources ckprompt.ResourceInput `json:"resources"` + Inputs ckprompt.PromptInputs `json:"inputs"` +} + type session struct { ID string Campaign string @@ -81,10 +88,11 @@ func loadApplicationConfig(ctx context.Context, selection ckconfig.Selection, in if input != nil { campaignInput = input.Campaign } - key := applicationCacheKey{config: selection.Config, overlay: selection.Overlay, campaign: campaignInput} if cache != nil { - if cached, ok := cache.applications[key]; ok { - return cached, nil + for _, cached := range cache.applications { + if reflect.DeepEqual(cached.selection, selection) && cached.campaign == campaignInput { + return cached.result, nil + } } } source := configkit.Mapping("composed application settings", map[string]any{ @@ -105,12 +113,10 @@ func loadApplicationConfig(ctx context.Context, selection ckconfig.Selection, in if campaignInput.Set { options.Overrides = []configkit.Override{{Path: configkit.Keys("default_campaign"), Value: campaignInput.Value, Flag: "--campaign"}} } + options.Overrides = append(options.Overrides, selection.Overrides...) result, err := configkit.Load(source, options) if err == nil && result != nil && cache != nil { - if cache.applications == nil { - cache.applications = make(map[applicationCacheKey]*configkit.Result[appConfiguration]) - } - cache.applications[key] = result + cache.applications = append(cache.applications, applicationCacheEntry{selection: selection, campaign: campaignInput, result: result}) } return result, err } @@ -224,6 +230,7 @@ func newAppWithEngineLoader( stateFactory := func() *runCache { return &runCache{llmClient: client} } appScope := ckconfig.FromConfigKit("app", "Application settings.", loadApplicationConfig) + appScope.Features.Overrides = true engineScope := ckprompt.ConfigScope(engineLoader) configCommand := ckconfig.Command(ckconfig.Suite[*inspectionInput, *runCache]{ NewInput: inputFactory, @@ -246,11 +253,28 @@ func newAppWithEngineLoader( definitions.NewState = stateFactory definitions.Selectors = []commandkit.Flag[*inspectionInput]{campaignSelector()} definitions = ckprompt.WithRequestValidation(definitions, engineLoader, buildValidationRequest) + definitions = ckprompt.WithRequestPreparation(definitions, engineLoader, + func(ctx context.Context, seed ckprompt.PreparationRequest, input *inspectionInput, cache *runCache) (promptkit.RunRequest, error) { + return buildValidationRequest(ctx, ckprompt.ValidationRequest{Selection: seed.Selection, Request: seed.Request, Namespaces: seed.Namespaces}, input, cache) + }) definitions = ckprompt.WithSmokeTesting(definitions, engineLoader, nil) + definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{}) if smokeTest != nil { definitions.SmokeTest = smokeTest } definitionDeclarations := ckprompt.Declarations(definitions) + resources := ckprompt.NewResourceFlags(func(input *requestInputs) *ckprompt.ResourceInput { return &input.Resources }) + execution := ckprompt.NewExecutionFlags(func(input *requestInputs) *ckprompt.ExecutionInput { return &input.Execution }) + inputs := ckprompt.NewPromptInputFlags(func(input *requestInputs) *ckprompt.PromptInputs { return &input.Inputs }) + requestFlags := append([]commandkit.Flag[requestInputs]{resources.Prompt, resources.Profile}, execution.All()...) + requestFlags = append(requestFlags, inputs.All()...) + requestInspect := commandkit.NewCommand(commandkit.CommandSpec[requestInputs]{ + Name: "inspect", Summary: "Inspect application-owned request inputs.", Flags: requestFlags, + Run: func(invocation commandkit.Invocation, input requestInputs) commandkit.Outcome { + _ = invocation.Reporter().Report(commandkit.Report{Value: input}) + return commandkit.Success() + }, + }) store := map[string]session{ "session-42": {ID: "session-42", Campaign: "spring", Summary: "A synthetic planning session."}, @@ -294,6 +318,7 @@ func newAppWithEngineLoader( configCommand, commandkit.Completion("composed-example"), commandkit.Group("session", "Manage saved sessions.", restore), + commandkit.Group("request", "Inspect application-owned request inputs.", requestInspect), commandkit.Alias("list collections", "promptkit list collections"), commandkit.Alias("promptkit list bundles", "promptkit list collections"), commandkit.Alias("pk", "promptkit"), diff --git a/integrations/ckprompt/examples/composed/main_test.go b/integrations/ckprompt/examples/composed/main_test.go index 963572d..e998c51 100644 --- a/integrations/ckprompt/examples/composed/main_test.go +++ b/integrations/ckprompt/examples/composed/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -101,6 +102,159 @@ func TestApplicationCacheDoesNotRetainFailedLoads(t *testing.T) { } } +func TestApplicationCacheDistinguishesOverrides(t *testing.T) { + cache := &runCache{} + selection := func(value string) ckconfig.Selection { + return ckconfig.Selection{Overrides: []configkit.Override{{Path: configkit.Keys("campaigns", "spring", "model"), Value: value, Flag: "--set"}}} + } + left, err := loadApplicationConfig(context.Background(), selection("left"), nil, cache) + if err != nil { + t.Fatal(err) + } + right, err := loadApplicationConfig(context.Background(), selection("right"), nil, cache) + if err != nil { + t.Fatal(err) + } + again, err := loadApplicationConfig(context.Background(), selection("left"), nil, cache) + if err != nil { + t.Fatal(err) + } + if left == right || left != again || left.Config.Campaigns["spring"].Model != "left" || right.Config.Campaigns["spring"].Model != "right" { + t.Fatalf("override selections reused the wrong result: left=%#v right=%#v", left.Config, right.Config) + } +} + +func TestComposedIndependentOverridesAndFailureEvidence(t *testing.T) { + app, err := newApp() + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + args := []string{"config", "diff", "--scope", "app", "--campaign", "spring", "--limit=0", "--against-campaign", "autumn", "--against-limit=0", "--set", `/campaigns/spring/model="left"`, "--against-set", `/campaigns/spring/model="right"`, "--details", "--format", "json"} + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("comparison: %#v, %s", outcome, output.String()) + } + var report struct { + Differences []struct { + Path string `json:"path"` + Before struct { + Value any `json:"value"` + } `json:"before"` + After struct { + Value any `json:"value"` + } `json:"after"` + } `json:"differences"` + } + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatal(err) + } + changes := map[string][2]any{} + for _, difference := range report.Differences { + changes[difference.Path] = [2]any{difference.Before.Value, difference.After.Value} + } + if changes["/campaigns/spring/model"] != [2]any{"left", "right"} || changes["/default_campaign"] != [2]any{"spring", "autumn"} { + t.Fatalf("independent comparison: %#v", changes) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"config", "sources", "--scope", "app", "--config", "fixtures/missing.yml", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var evidence struct { + Sources []any `json:"sources"` + } + if err := json.Unmarshal(output.Bytes(), &evidence); err != nil { + t.Fatal(err) + } + if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || len(evidence.Sources) == 0 { + t.Fatalf("failed load evidence: %#v, %s", outcome, output.String()) + } + + outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil { + t.Fatalf("overlapping native overrides: %#v", outcome) + } + loads := 0 + loader := func(ctx context.Context, selection ckconfig.Selection, input *inspectionInput, cache *runCache) (*promptkit.Engine, error) { + loads++ + return loadPromptEngine(ctx, selection, input, cache) + } + app, err = newAppWithEngineLoader(&exampleLLMClient{}, loader, nil) + if err != nil { + t.Fatal(err) + } + outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "all", "--set", `/default_campaign="spring"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || loads != 0 { + t.Fatalf("unsupported aggregate override: %#v", outcome) + } +} + +func TestComposedPreviewAndReusableFlags(t *testing.T) { + calls := 0 + app, err := newAppWithLLMClient(&exampleLLMClient{calls: &calls}) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + args := []string{"promptkit", "show", "prepared", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt", "--input", "topic=sprint", "--profile", "spring", "--extra-params", `{"campaign":"preview"}`, "--format", "json"} + outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess { + t.Fatalf("preview: %#v, %s", outcome, output.String()) + } + var report promptkit.PreparedRun + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatal(err) + } + input := &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "spring", Set: true}} + engine, err := loadPromptEngine(context.Background(), ckconfig.Selection{}, input, nil) + if err != nil { + t.Fatal(err) + } + request, err := buildValidationRequest(context.Background(), ckprompt.ValidationRequest{Request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "spring", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("sprint")}, Execution: &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{"campaign": "preview"}}}}, input, nil) + if err != nil { + t.Fatal(err) + } + native, err := engine.Prepare(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if report.PromptID != native.PromptID || !reflect.DeepEqual(report.Messages, native.Messages) || !reflect.DeepEqual(report.EffectiveModelParams, native.EffectiveModelParams) || calls != 0 { + t.Fatalf("preview diverged from native result or called model: %#v", report) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--prompt-namespace", "field-notes", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var namespaced promptkit.PromptInspection + if err := json.Unmarshal(output.Bytes(), &namespaced); err != nil { + t.Fatal(err) + } + view, err := engine.WithNamespaces(promptkit.Namespaces{Prompt: "field-notes"}) + if err != nil { + t.Fatal(err) + } + wantPrompt, err := view.InspectPrompt(context.Background(), "briefing") + if err != nil { + t.Fatal(err) + } + if outcome.Kind != commandkit.OutcomeSuccess || namespaced.PromptID != wantPrompt.PromptID { + t.Fatalf("namespace selection: %#v, %#v", outcome, namespaced) + } + + output.Reset() + args = []string{"request", "inspect", "--llm-prompt", "briefing", "--llm-profile=", "--llm-model", "sample", "--llm-input-text", "topic=hello", "--format", "json"} + outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) + var owned requestInputs + if err := json.Unmarshal(output.Bytes(), &owned); err != nil { + t.Fatal(err) + } + if outcome.Kind != commandkit.OutcomeSuccess || !owned.Resources.Profile.Set || owned.Resources.Profile.Value != "" || owned.Execution.Execution.Model != "sample" || owned.Inputs.Inputs["topic"] != promptkit.Inline("hello") { + t.Fatalf("application flags: %#v, %#v", outcome, owned) + } + outcome = app.Execute(context.Background(), []string{"request", "inspect", "--model", "sample"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage { + t.Fatalf("unexpected application alias: %#v", outcome) + } +} + func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) { app, err := newApp() if err != nil { @@ -266,15 +420,19 @@ func TestBuildValidationRequestDefaultsOnlyAbsentVariables(t *testing.T) { if !reflect.DeepEqual(got.Inputs, inputs) || !reflect.DeepEqual(seed.Request.Vars, explicit) { t.Fatalf("seed bindings changed: built=%#v seed=%#v", got.Inputs, explicit) } - key := applicationCacheKey{config: seed.Selection.Config, overlay: seed.Selection.Overlay, campaign: input.Campaign} - cached := cache.applications[key] + var cached *configkit.Result[appConfiguration] + for _, entry := range cache.applications { + if reflect.DeepEqual(entry.selection, seed.Selection) && entry.campaign == input.Campaign { + cached = entry.result + } + } if cached == nil { t.Fatal("builder did not load application configuration into invocation state") } if _, err := loadPromptEngine(context.Background(), seed.Selection, input, cache); err != nil { t.Fatal(err) } - if cache.applications[key] != cached { + if len(cache.applications) != 1 || cache.applications[0].result != cached { t.Fatal("engine loader did not reuse the builder's application configuration") } @@ -518,6 +676,11 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) { if outcome.Kind != commandkit.OutcomeSuccess || output.Len() != 0 || engineLoads != 0 || clientCalls != 0 { t.Fatalf("quiet aliased help outcome/output/engine loads/client calls = (%#v, %q, %d, %d)", outcome, output.String(), engineLoads, clientCalls) } + output.Reset() + outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || engineLoads != 0 || clientCalls != 0 { + t.Fatalf("version outcome/engine loads/client calls = (%#v, %d, %d)", outcome, engineLoads, clientCalls) + } for _, args := range [][]string{ {"list", "collections", "--campaign", "spring"},