diff --git a/docs/cli.md b/docs/cli.md index 9af8147..a6f8240 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -563,8 +563,8 @@ native selection and discovery rules: ```text example promptkit requirements show briefing --campaign spring -example promptkit requirements show briefing --profile editorial --var mode=full -example promptkit requirements show briefing --session-id session-42 --var note= --format yaml +example promptkit requirements show briefing --llm-profile editorial --llm-var mode=full +example promptkit requirements show briefing --llm-session-id session-42 --llm-var note= --format yaml ``` The command accepts: @@ -574,9 +574,9 @@ The command accepts: | `--config` | path string | omitted | Select the configuration input; explicit presence is retained. | | `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. | | Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. | -| `--profile` | string | empty | Pass the profile ID to PromptKit; empty uses its native selection rule. | -| `--session-id` | string | empty | Pass session context to PromptKit. | -| `--var` | repeatable `KEY=VALUE` | none | Supply caller-variable context for native discovery. | +| `--llm-profile` | string | empty | Pass the profile ID to PromptKit; empty uses its native selection rule. | +| `--llm-session-id` | string | empty | Pass session context to PromptKit. | +| `--llm-var` | repeatable `KEY=VALUE` | none | Supply caller-variable context for native discovery. | | `--format` | `text`, `yaml`, `json` | automatic | Select the requirements report format; automatic and text use YAML for this structured report. | The `promptkit` namespace identifies the operation, so there is no built-in @@ -585,10 +585,10 @@ native empty-string meaning. Requirements inspection does not require a profile to be execution-ready. See the [PromptKit v0.18.0 consumer guide](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/d9f6aad30509e538fc73ea16710df3a3ec084502/docs/consumers/pkg-promptkit.md) for native configuration, selection, and requirements behavior. The built-in -flags other than `--var` are singleton bindings; when repeated, the last value +flags other than `--llm-var` are singleton bindings; when repeated, the last value is used. Suite selectors retain their declared binding behavior. -Each `--var` occurrence must contain `=`. CommandKit splits at the first +Each `--llm-var` occurrence must contain `=`. CommandKit splits at the first separator and preserves the exact key and remaining value. Empty keys and values, whitespace, commas, and additional `=` characters are retained. All occurrences are parsed; duplicate keys use the last value, while any malformed @@ -644,8 +644,8 @@ prompt ID and passes a concrete request to PromptKit's native `Engine.ValidateRequest` preparation path: ```text -example promptkit requirements validate briefing --campaign spring --input topic=topic.txt -example promptkit requirements validate briefing --profile editorial --session-id session-42 --var mode=full --input notes=notes.txt +example promptkit requirements validate briefing --campaign spring --llm-input-file topic=topic.txt +example promptkit requirements validate briefing --llm-profile editorial --llm-session-id session-42 --llm-var mode=full --llm-input-file notes=notes.txt ``` The command accepts: @@ -655,24 +655,28 @@ The command accepts: | `--config` | path string | omitted | Select the configuration input; explicit presence is retained. | | `--overlay` | path string | omitted | Select one overlay; explicit presence is retained. Repeated occurrences use the last value. | | Suite selectors | application-defined | per selector | The flags captured from `Suite.Selectors`. | -| `--profile` | string | empty | Pass the profile ID to PromptKit. | -| `--session-id` | string | empty | Pass session context to PromptKit. | -| `--var` | repeatable `KEY=VALUE` | none | Supply caller variables for the concrete request. | -| `--input` | repeatable `NAME=PATH` | none | Bind a file-backed PromptKit artifact. | +| `--llm-profile` | string | empty | Pass the profile ID to PromptKit. | +| `--llm-session-id` | string | empty | Pass session context to PromptKit. | +| `--llm-var` | repeatable `KEY=VALUE` | none | Supply caller variables for the concrete request. | +| `--llm-input-text` | repeatable `NAME=TEXT` | none | Bind a literal PromptKit artifact. | +| `--llm-input-file` | repeatable `NAME=PATH` | none | Bind a file-backed PromptKit artifact. | | `--format` | `text`, `yaml`, `json` | automatic | Select the success confirmation format; automatic and text use YAML for this structured report. | -The built-in flags other than repeatable `--var` and `--input` are singleton -bindings; when repeated, the last value is used. Explicit empty `--config` or +Validation also accepts the execution controls from `--llm-endpoint` through +`--llm-extra-params` listed under [smoke testing](#promptkit-smoke-testing). +The built-in flags other than +`--llm-var`, `--llm-input-text`, and `--llm-input-file` are singleton bindings; +when repeated, the last value is used. Explicit empty `--config` or `--overlay` values keep their supplied-presence information. The profile and session fields use native empty-string semantics. PromptKit owns their interpretation and selection errors. -Each `--var` and `--input` occurrence must contain `=`. CommandKit splits at the -first separator and preserves the exact key/name and remaining value/path. -Empty variable keys and values and empty input names and paths are passed to -native validation. File paths become `promptkit.File(PATH)` references without -being read, normalized, expanded, or rebased during argument binding. Duplicate -keys or names use the last value, but every occurrence is parsed. +Each `--llm-var`, `--llm-input-text`, and `--llm-input-file` occurrence must +contain `=`. CommandKit splits at the first separator and preserves the exact +key/name and remaining value. Empty names and values are passed to native +validation. Text becomes a `promptkit.Inline` reference; file paths become +`promptkit.File` references without being read, normalized, expanded, or +rebased during argument binding. Every occurrence is parsed. On success, the confirmation contains only the requested prompt ID and `valid: true`: @@ -765,33 +769,39 @@ each aggregate call changes the seed kind before invoking it. The command accepts: ```text -example promptkit test text_inference --campaign spring --prompt briefing --profile spring -example promptkit test all --campaign spring --prompt briefing --profile spring -example promptkit test authentication --backend local +example promptkit test text_inference --campaign spring --llm-prompt briefing --llm-profile spring +example promptkit test all --campaign spring --llm-prompt briefing --llm-profile spring +example promptkit test authentication --llm-backend local ``` | Flag | Values | Default | Meaning | | --- | --- | --- | --- | | `--config`, `--overlay` | path strings | omitted | Select PromptKit configuration; supplied presence is retained. | | Suite selectors | application-defined | per selector | Captured flags from `Suite.Selectors`. | -| `--backend` | string | empty | Native backend ID. | -| `--prompt` | string | empty | Native prompt ID. | -| `--profile` | string | empty | Native profile ID. | -| `--endpoint` | string | omitted | Override the execution endpoint. | -| `--model` | string | omitted | Override the execution model. | -| `--temperature` | float | omitted | Override execution temperature. | -| `--max-tokens` | integer | omitted | Override execution maximum tokens. | -| `--top-p` | float | omitted | Override execution top-p. | -| `--timeout-seconds` | integer | omitted | Override the execution timeout. | -| `--service-tier` | string | omitted | Override the execution service tier. | -| `--reasoning-effort` | string | omitted | Override execution reasoning effort. | -| `--api-key-env` | string | omitted | Supply an API-key environment-variable name to native configuration. | -| `--transient-retries` | integer | omitted | Set the native inference transient retry count. | +| `--llm-backend` | string | empty | Native backend ID. | +| `--llm-prompt` | string | empty | Native prompt ID. | +| `--llm-profile` | string | empty | Native profile ID. | +| `--llm-endpoint` | string | omitted | Override the execution endpoint. | +| `--llm-model` | string | omitted | Override the execution model. | +| `--llm-temperature` | float | omitted | Override execution temperature. | +| `--llm-max-tokens` | integer | omitted | Override execution maximum tokens. | +| `--llm-top-p` | float | omitted | Override execution top-p. | +| `--llm-timeout-seconds` | integer | omitted | Override the execution timeout. | +| `--llm-service-tier` | string | omitted | Override the execution service tier. | +| `--llm-reasoning-effort` | string | omitted | Override execution reasoning effort. | +| `--llm-api-key-env` | string | omitted | Supply an API-key environment-variable name to native configuration. | +| `--llm-transient-retries` | integer | omitted | Set the native inference transient retry count. | +| `--llm-extra-params` | JSON object | omitted | Supply provider parameters as a whole map. | | `--format` | `text`, `yaml`, `json` | automatic | Select the report format; automatic and text use YAML for these structured reports. | The execution flags populate PromptKit's native execution override. The -`--api-key-env` value is a variable name, not a credential value. Application -selectors, loader policy, credentials, and client wiring remain application +`--llm-api-key-env` value is a variable name, not a credential value. The +extra-parameters flag accepts one JSON object, including an empty object; +PromptKit owns its native replacement semantics and validation. Integer JSON +numbers use signed or unsigned 64-bit integers where representable; decimal +and exponent numbers use floating point. + +Application selectors, loader policy, credentials, and client wiring remain application owned. The `WithSmokeTesting` helper accepts an optional typed builder. A nil builder forwards the seed; a supplied builder's complete native request is authoritative. The helper retains a native report when PromptKit returns both a diff --git a/docs/consumers/guide.md b/docs/consumers/guide.md index 470888c..2832505 100644 --- a/docs/consumers/guide.md +++ b/docs/consumers/guide.md @@ -326,6 +326,12 @@ and state. `FromEngine` uses PromptKit's public listing and inspection methods, including `Engine.InspectRequirements`; validation must be opted into separately and uses `Engine.ValidateRequest`. +Application-owned commands can use `ckprompt.NewExecutionFlags`, +`ckprompt.NewResourceFlags`, and `ckprompt.NewPromptInputFlags` to bind the +same canonical request flags into their own input types. Each collection exposes +individual handles and an `All` method, so a command can choose its applicable +subset. The application decides how to apply the resulting values to its request. + Catalog, named definition inspection, and requirements discovery do not render messages or load request artifacts. Concrete request validation follows native PromptKit preparation, which can resolve prompt/profile resources, load diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 104c10d..f0ab27b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Shared reporting and PromptKit v0.20.0 implementation plan -Status: implementation in progress; Stages 1–9 complete. +Status: implementation in progress; Stages 1–10 complete. ## Scope, authority, and execution @@ -350,6 +350,11 @@ for the converted commands. ## Stage 10: Reusable execution and resource flag collections +**Complete.** Reusable request flag collections feed requirements discovery, +validation, and smoke requests; the next stage supplies ordered text/file +precedence and prepared-request preview. The root, ckconfig, and ckprompt +workspace suites passed with Go 1.27.0, including collection and builder tests. + Implement CLI-specific collections in ckprompt using ordinary public core flags. Use typed flag-set structs exposing named original `Flag[I]` handles and an `All() []Flag[I]` convenience method; callers select subsets with ordinary slices. diff --git a/integrations/ckprompt/command.go b/integrations/ckprompt/command.go index de68dac..1e29c1a 100644 --- a/integrations/ckprompt/command.go +++ b/integrations/ckprompt/command.go @@ -31,6 +31,8 @@ type catalogInput[I any] struct { type requirementsInput[I any] struct { selection ckconfig.Selection inspection promptkit.RequirementInspectionRequest + resources ResourceInput + inputs PromptInputs captured commandkit.FlagValues[I] selected I } @@ -281,24 +283,12 @@ func newRequirementsCommand[I, S any](suite Suite[I, S]) commandkit.Command { func(input **requirementsInput[I], value commandkit.Supplied[string]) { (*input).selection.Overlay = value }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "Profile identity for discovery."}, stringDefault(""), parseDefinitionString, - func(input **requirementsInput[I], value commandkit.Supplied[string]) { - (*input).inspection.ProfileID = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "session-id", Usage: "Session context for discovery."}, stringDefault(""), parseDefinitionString, - func(input **requirementsInput[I], value commandkit.Supplied[string]) { - (*input).inspection.SessionID = value.Value - }), - commandkit.RepeatFlag(commandkit.FlagMeta{Name: "var", Usage: "Caller variable assignment KEY=VALUE."}, []variableAssignment(nil), parseVariableAssignment, - func(input **requirementsInput[I], value commandkit.Supplied[[]variableAssignment]) { - variables := make(map[string]string, len(value.Value)) - for _, assignment := range value.Value { - variables[assignment.key] = assignment.value - } - (*input).inspection.Vars = variables - }), ) + resources := NewResourceFlags(func(input **requirementsInput[I]) *ResourceInput { return &(*input).resources }) + inputs := NewPromptInputFlags(func(input **requirementsInput[I]) *PromptInputs { return &(*input).inputs }) + flags = append(flags, resources.Profile, resources.SessionID, inputs.Variable) + spec := commandkit.CommandSpec[*requirementsInput[I]]{ Name: "show", Summary: "Inspect requirements for a named PromptKit prompt.", @@ -313,6 +303,9 @@ func newRequirementsCommand[I, S any](suite Suite[I, S]) commandkit.Command { }), }, CheckInput: func(input *requirementsInput[I]) error { + input.inspection.ProfileID = input.resources.Profile.Value + input.inspection.SessionID = input.resources.SessionID.Value + input.inspection.Vars = input.inputs.Vars selected, err := prepareSuiteInput(suite, input.selection, input.captured) if err != nil { return err diff --git a/integrations/ckprompt/command_test.go b/integrations/ckprompt/command_test.go index 33f406a..38b330e 100644 --- a/integrations/ckprompt/command_test.go +++ b/integrations/ckprompt/command_test.go @@ -141,7 +141,7 @@ func TestRequirementsCommandCapturesContextAndVariableAssignments(t *testing.T) if familyOutcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(familyHelp.String(), "requirements") || strings.Contains(familyHelp.String(), "\n show ") || strings.Contains(familyHelp.String(), "\n list ") { t.Fatalf("requirements-only family help = (%#v, %q)", familyOutcome, familyHelp.String()) } - for _, flag := range []string{"--config", "--overlay", "--profile", "--session-id", "--var", "--format", "--campaign"} { + for _, flag := range []string{"--config", "--overlay", "--llm-profile", "--llm-session-id", "--llm-var", "--format", "--campaign"} { if !strings.Contains(help.String(), flag) { t.Errorf("requirements help omits %s: %q", flag, help.String()) } @@ -154,10 +154,10 @@ func TestRequirementsCommandCapturesContextAndVariableAssignments(t *testing.T) args := []string{ "promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--config", "before.yml", "--config=", "--overlay", "base.yml", "--overlay=", - "--profile", "first", "--profile", "editorial", - "--session-id", "old-session", "--session-id", "session-42", "--format", "text", "--format", "yaml", - "--var", "note=hello, world", "--var", "empty=", "--var", "token=a=b", "--var", "=value", - "--var", "mode=brief", "--var", "mode=full", + "--llm-profile", "first", "--llm-profile", "editorial", + "--llm-session-id", "old-session", "--llm-session-id", "session-42", "--format", "text", "--format", "yaml", + "--llm-var", "note=hello, world", "--llm-var", "empty=", "--llm-var", "token=a=b", "--llm-var", "=value", + "--llm-var", "mode=brief", "--llm-var", "mode=full", } outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 { @@ -203,9 +203,9 @@ func TestRequirementsCommandUsageFailuresPrecedeApplicationPreparation(t *testin {name: "missing operand", args: []string{"promptkit", "requirements", "show"}}, {name: "extra operand", args: []string{"promptkit", "requirements", "show", "briefing", "extra"}}, {name: "unsupported format", args: []string{"promptkit", "requirements", "show", "briefing", "--format", "invalid"}}, - {name: "malformed variable followed by valid assignment", args: []string{"promptkit", "requirements", "show", "briefing", "--var", "broken", "--var", "mode=full"}}, + {name: "malformed variable followed by valid assignment", args: []string{"promptkit", "requirements", "show", "briefing", "--llm-var", "broken", "--llm-var", "mode=full"}}, {name: "no built-in scope", args: []string{"promptkit", "requirements", "show", "briefing", "--scope", "all"}}, - {name: "malformed variable on help", args: []string{"help", "promptkit", "requirements", "show", "--var", "broken"}}, + {name: "malformed variable on help", args: []string{"help", "promptkit", "requirements", "show", "--llm-var", "broken"}}, } { t.Run(test.name, func(t *testing.T) { newInputCalls, checkInputCalls, newStateCalls, providerCalls := 0, 0, 0, 0 diff --git a/integrations/ckprompt/examples/composed/README.md b/integrations/ckprompt/examples/composed/README.md index 3696aec..01567a6 100644 --- a/integrations/ckprompt/examples/composed/README.md +++ b/integrations/ckprompt/examples/composed/README.md @@ -25,13 +25,13 @@ go run . promptkit show prompt briefing --campaign spring --format yaml go run . pk show prompt briefing --campaign spring --fmt yaml go run . promptkit show profile spring --campaign spring go run . promptkit requirements show briefing --campaign spring -go run . promptkit requirements show briefing --campaign spring --var mode=full --format yaml -go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt -go run . promptkit requirements validate briefing --campaign spring --input topic=fixtures/topic.txt --var mode=full --input notes=fixtures/notes.txt --format yaml +go run . promptkit requirements show briefing --campaign spring --llm-var mode=full --format yaml +go run . promptkit requirements validate briefing --campaign spring --llm-input-file topic=fixtures/topic.txt +go run . promptkit requirements validate briefing --campaign spring --llm-input-file topic=fixtures/topic.txt --llm-var mode=full --llm-input-file notes=fixtures/notes.txt --format yaml go run . promptkit requirements validate briefing --campaign spring -go run . promptkit test text_inference --campaign spring --prompt briefing --profile spring -go run . smoke text_inference --campaign spring --prompt briefing --profile spring -go run . promptkit test all --campaign spring --prompt briefing --profile spring +go run . promptkit test text_inference --campaign spring --llm-prompt briefing --llm-profile spring +go run . smoke text_inference --campaign spring --llm-prompt briefing --llm-profile spring +go run . promptkit test all --campaign spring --llm-prompt briefing --llm-profile spring go run . pk --hlep go run . session restore session-42 --campaign spring ``` diff --git a/integrations/ckprompt/examples/composed/main_test.go b/integrations/ckprompt/examples/composed/main_test.go index 1a9ea75..1e438af 100644 --- a/integrations/ckprompt/examples/composed/main_test.go +++ b/integrations/ckprompt/examples/composed/main_test.go @@ -164,7 +164,7 @@ func TestComposedRequirementsDiscoveryUsesSelectedCampaign(t *testing.T) { args []string }{ {name: "without variables", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}}, - {name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--var", "mode=full", "--format", "yaml"}}, + {name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--llm-var", "mode=full", "--format", "yaml"}}, } { t.Run(test.name, func(t *testing.T) { var output bytes.Buffer @@ -204,11 +204,11 @@ func TestComposedRequestValidationUsesFileBindings(t *testing.T) { }{ { name: "default mode", - args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt"}, + args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt"}, }, { name: "conditional notes", - args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt", "--var", "mode=full", "--input", "notes=fixtures/notes.txt", "--format", "yaml"}, + args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "topic=fixtures/topic.txt", "--llm-var", "mode=full", "--llm-input-file", "notes=fixtures/notes.txt", "--format", "yaml"}, }, } { t.Run(test.name, func(t *testing.T) { @@ -393,7 +393,7 @@ func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T) } var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) { t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String()) } @@ -453,7 +453,7 @@ func TestComposedSmokeReportSurvivesInferenceFailure(t *testing.T) { t.Fatal(err) } var output, diagnostics bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "text_inference", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, wantCause) { t.Fatalf("smoke failure outcome = %#v; report %q; diagnostics %q", outcome, output.String(), diagnostics.String()) } @@ -543,7 +543,7 @@ func TestComposedAliasesReachCanonicalCommands(t *testing.T) { } output.Reset() - outcome = app.Execute(context.Background(), []string{"smoke", "text_inference", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) + outcome = app.Execute(context.Background(), []string{"smoke", "text_inference", "--campaign", "spring", "--llm-prompt", "briefing", "--llm-profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "kind: text_inference") || !strings.Contains(output.String(), "completed: true") || clientCalls != 1 { t.Fatalf("smoke shortcut outcome/report/client calls = (%#v, %q, %d)", outcome, output.String(), clientCalls) } diff --git a/integrations/ckprompt/request_flags.go b/integrations/ckprompt/request_flags.go new file mode 100644 index 0000000..7a37334 --- /dev/null +++ b/integrations/ckprompt/request_flags.go @@ -0,0 +1,196 @@ +package ckprompt + +import ( + "fmt" + "strconv" + + commandkit "go.maximumdirect.net/commandkit" + promptkit "go.maximumdirect.net/promptkit" +) + +// requestFlagDefinition keeps the canonical spellings, help text, and optional +// command-scoped aliases together for consumers of the standard flag sets. +type requestFlagDefinition struct { + meta commandkit.FlagMeta + aliases []string +} + +var requestFlags = struct { + Endpoint requestFlagDefinition + Model requestFlagDefinition + Temperature requestFlagDefinition + MaxTokens requestFlagDefinition + TopP requestFlagDefinition + TimeoutSeconds requestFlagDefinition + ServiceTier requestFlagDefinition + ReasoningEffort requestFlagDefinition + APIKeyEnv requestFlagDefinition + TransientRetries requestFlagDefinition + ExtraParams requestFlagDefinition + Prompt requestFlagDefinition + Profile requestFlagDefinition + Backend requestFlagDefinition + SessionID requestFlagDefinition + Var requestFlagDefinition + InputText requestFlagDefinition + InputFile requestFlagDefinition +}{ + Endpoint: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-endpoint", Usage: "Execution endpoint override."}, []string{"endpoint"}}, + Model: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-model", Usage: "Execution model override."}, []string{"model"}}, + Temperature: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-temperature", Usage: "Execution temperature override."}, []string{"temperature"}}, + MaxTokens: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-max-tokens", Usage: "Execution maximum tokens override."}, []string{"max-tokens"}}, + TopP: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-top-p", Usage: "Execution top-p override."}, []string{"top-p"}}, + TimeoutSeconds: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-timeout-seconds", Usage: "Execution timeout override in seconds."}, []string{"timeout-seconds"}}, + ServiceTier: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-service-tier", Usage: "Execution service tier."}, []string{"service-tier"}}, + ReasoningEffort: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-reasoning-effort", Usage: "Execution reasoning effort override."}, []string{"reasoning-effort"}}, + APIKeyEnv: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-api-key-env", Usage: "Execution API key environment variable."}, []string{"api-key-env"}}, + TransientRetries: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-transient-retries", Usage: "Inference transient retry count."}, []string{"transient-retries"}}, + ExtraParams: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-extra-params", Usage: "Execution provider parameters as a JSON object."}, []string{"extra-params"}}, + Prompt: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-prompt", Usage: "PromptKit prompt ID."}, []string{"prompt"}}, + Profile: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-profile", Usage: "PromptKit profile ID."}, []string{"profile"}}, + Backend: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-backend", Usage: "PromptKit backend ID."}, []string{"backend"}}, + SessionID: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-session-id", Usage: "PromptKit session ID."}, []string{"session-id"}}, + Var: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-var", Usage: "Caller variable assignment KEY=VALUE."}, []string{"var"}}, + InputText: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-input-text", Usage: "Literal prompt input assignment NAME=TEXT."}, []string{"input-text", "llm-input", "input"}}, + InputFile: requestFlagDefinition{commandkit.FlagMeta{Name: "llm-input-file", Usage: "Prompt input file assignment NAME=PATH."}, []string{"input-file"}}, +} + +// ExecutionInput contains request-scoped execution controls supplied on the CLI. +// A nil override or retry count leaves the corresponding native default intact. +type ExecutionInput struct { + Execution *promptkit.ExecutionTargetOverride + TransientRetries *int +} + +// ExecutionFlags exposes independently selectable execution-control bindings. +type ExecutionFlags[I any] struct { + Endpoint, Model, Temperature, MaxTokens, TopP, TimeoutSeconds commandkit.Flag[I] + ServiceTier, ReasoningEffort, APIKeyEnv, TransientRetries, ExtraParams commandkit.Flag[I] +} + +// All returns all execution controls. +func (f ExecutionFlags[I]) All() []commandkit.Flag[I] { + return []commandkit.Flag[I]{f.Endpoint, f.Model, f.Temperature, f.MaxTokens, f.TopP, f.TimeoutSeconds, f.ServiceTier, f.ReasoningEffort, f.APIKeyEnv, f.TransientRetries, f.ExtraParams} +} + +// NewExecutionFlags binds execution controls to a field of the caller's input. +func NewExecutionFlags[I any](destination func(*I) *ExecutionInput) ExecutionFlags[I] { + target := func(input *I) *promptkit.ExecutionTargetOverride { + value := destination(input) + if value.Execution == nil { + value.Execution = &promptkit.ExecutionTargetOverride{} + } + return value.Execution + } + return ExecutionFlags[I]{ + Endpoint: executionFlag(requestFlags.Endpoint.meta, parseDefinitionString, func(i *I, v string) { target(i).Endpoint = v }), + Model: executionFlag(requestFlags.Model.meta, parseDefinitionString, func(i *I, v string) { target(i).Model = v }), + Temperature: executionFlag(requestFlags.Temperature.meta, parseExecutionFloat, func(i *I, v float64) { target(i).Temperature = &v }), + MaxTokens: executionFlag(requestFlags.MaxTokens.meta, strconv.Atoi, func(i *I, v int) { target(i).MaxTokens = &v }), + TopP: executionFlag(requestFlags.TopP.meta, parseExecutionFloat, func(i *I, v float64) { target(i).TopP = &v }), + TimeoutSeconds: executionFlag(requestFlags.TimeoutSeconds.meta, strconv.Atoi, func(i *I, v int) { target(i).TimeoutSeconds = &v }), + ServiceTier: executionFlag(requestFlags.ServiceTier.meta, parseDefinitionString, func(i *I, v string) { target(i).ServiceTier = v }), + ReasoningEffort: executionFlag(requestFlags.ReasoningEffort.meta, parseDefinitionString, func(i *I, v string) { target(i).ReasoningEffort = &v }), + APIKeyEnv: executionFlag(requestFlags.APIKeyEnv.meta, parseDefinitionString, func(i *I, v string) { target(i).APIKeyEnv = v }), + TransientRetries: executionFlag(requestFlags.TransientRetries.meta, strconv.Atoi, func(i *I, v int) { destination(i).TransientRetries = &v }), + ExtraParams: executionFlag(requestFlags.ExtraParams.meta, parseExtraParams, func(i *I, v map[string]any) { target(i).ExtraParams = v }), + } +} + +func executionFlag[I, V any](meta commandkit.FlagMeta, parse func(string) (V, error), set func(*I, V)) commandkit.Flag[I] { + return commandkit.ValueFlag(meta, (*V)(nil), parse, func(i *I, value commandkit.Supplied[V]) { + if value.Set { + set(i, value.Value) + } + }) +} + +func parseExtraParams(raw string) (map[string]any, error) { + value, err := commandkit.ParseCLIJSON(raw) + if err != nil { + return nil, err + } + object, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("provider parameters must be a JSON object") + } + return object, nil +} + +// ResourceInput retains whether each resource identifier was explicitly supplied. +type ResourceInput struct { + Prompt, Profile, Backend, SessionID commandkit.Supplied[string] +} + +// ResourceFlags exposes independently selectable resource and session bindings. +type ResourceFlags[I any] struct { + Prompt, Profile, Backend, SessionID commandkit.Flag[I] +} + +// All returns all resource bindings. +func (f ResourceFlags[I]) All() []commandkit.Flag[I] { + return []commandkit.Flag[I]{f.Prompt, f.Profile, f.Backend, f.SessionID} +} + +// NewResourceFlags binds resource identifiers to a field of the caller's input. +func NewResourceFlags[I any](destination func(*I) *ResourceInput) ResourceFlags[I] { + flag := func(meta commandkit.FlagMeta, selectField func(*ResourceInput) *commandkit.Supplied[string]) commandkit.Flag[I] { + return commandkit.ValueFlag(meta, (*string)(nil), parseDefinitionString, func(i *I, v commandkit.Supplied[string]) { *selectField(destination(i)) = v }) + } + return ResourceFlags[I]{ + Prompt: flag(requestFlags.Prompt.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Prompt }), + Profile: flag(requestFlags.Profile.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Profile }), + Backend: flag(requestFlags.Backend.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.Backend }), + SessionID: flag(requestFlags.SessionID.meta, func(v *ResourceInput) *commandkit.Supplied[string] { return &v.SessionID }), + } +} + +// PromptInputs contains caller variables and native artifact references. +type PromptInputs struct { + Vars map[string]string + Inputs map[string]promptkit.ArtifactRef +} + +// PromptInputFlags exposes variable and named artifact input bindings. +type PromptInputFlags[I any] struct { + Variable, Text, File commandkit.Flag[I] +} + +// All returns all variable and input bindings. +func (f PromptInputFlags[I]) All() []commandkit.Flag[I] { + return []commandkit.Flag[I]{f.Variable, f.Text, f.File} +} + +// NewPromptInputFlags binds caller variables and input references to a field of +// the caller's input. +func NewPromptInputFlags[I any](destination func(*I) *PromptInputs) PromptInputFlags[I] { + variable := commandkit.RepeatFlag(requestFlags.Var.meta, []variableAssignment(nil), parseVariableAssignment, func(i *I, v commandkit.Supplied[[]variableAssignment]) { + if !v.Set { + return + } + values := make(map[string]string, len(v.Value)) + for _, assignment := range v.Value { + values[assignment.key] = assignment.value + } + destination(i).Vars = values + }) + inputFlag := func(meta commandkit.FlagMeta, makeRef func(string) promptkit.ArtifactRef) commandkit.Flag[I] { + return commandkit.RepeatFlag(meta, []inputAssignment(nil), parseInputAssignment, func(i *I, v commandkit.Supplied[[]inputAssignment]) { + if !v.Set { + return + } + input := destination(i) + if input.Inputs == nil { + input.Inputs = make(map[string]promptkit.ArtifactRef) + } + for _, assignment := range v.Value { + input.Inputs[assignment.name] = makeRef(assignment.path) + } + }) + } + return PromptInputFlags[I]{Variable: variable, Text: inputFlag(requestFlags.InputText.meta, promptkit.Inline), File: inputFlag(requestFlags.InputFile.meta, promptkit.File)} +} + +func parseExecutionFloat(value string) (float64, error) { + return strconv.ParseFloat(value, 64) +} diff --git a/integrations/ckprompt/request_flags_test.go b/integrations/ckprompt/request_flags_test.go new file mode 100644 index 0000000..6dc44ec --- /dev/null +++ b/integrations/ckprompt/request_flags_test.go @@ -0,0 +1,78 @@ +package ckprompt_test + +import ( + "context" + "io" + "testing" + + "go.maximumdirect.net/ckprompt" + commandkit "go.maximumdirect.net/commandkit" + promptkit "go.maximumdirect.net/promptkit" +) + +func TestReusableRequestFlagsOnApplicationCommand(t *testing.T) { + type input struct { + execution ckprompt.ExecutionInput + resources ckprompt.ResourceInput + prompt ckprompt.PromptInputs + } + execution := ckprompt.NewExecutionFlags(func(i *input) *ckprompt.ExecutionInput { return &i.execution }) + resources := ckprompt.NewResourceFlags(func(i *input) *ckprompt.ResourceInput { return &i.resources }) + inputs := ckprompt.NewPromptInputFlags(func(i *input) *ckprompt.PromptInputs { return &i.prompt }) + flags := append(execution.All(), resources.Profile, inputs.Variable) + var received input + calls := 0 + command := commandkit.NewCommand(commandkit.CommandSpec[input]{ + Name: "run", Flags: flags, + Run: func(_ commandkit.Invocation, i input) commandkit.Outcome { + calls++ + received = i + return commandkit.Success() + }, + }) + app, err := commandkit.New(commandkit.Identity{Name: "app"}, command) + if err != nil { + t.Fatal(err) + } + run := func(args ...string) commandkit.Outcome { + return app.Execute(context.Background(), append([]string{"run"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard}) + } + if result := run(); result.Kind != commandkit.OutcomeSuccess || received.execution.Execution != nil || received.execution.TransientRetries != nil || received.resources.Profile.Set { + t.Fatalf("absent values: %#v %#v", result, received) + } + if result := run("--llm-profile=", "--llm-temperature=0", "--llm-reasoning-effort=", "--llm-transient-retries=0", "--llm-extra-params={\"n\":1,\"d\":1.5}", "--llm-var=x="); result.Kind != commandkit.OutcomeSuccess { + t.Fatal(result) + } + e := received.execution + if !received.resources.Profile.Set || received.resources.Profile.Value != "" || e.Execution == nil || e.Execution.Temperature == nil || *e.Execution.Temperature != 0 || e.Execution.ReasoningEffort == nil || *e.Execution.ReasoningEffort != "" || e.TransientRetries == nil || *e.TransientRetries != 0 || e.Execution.ExtraParams["n"] != int64(1) || e.Execution.ExtraParams["d"] != float64(1.5) || received.prompt.Vars["x"] != "" { + t.Fatalf("explicit values: %#v", received) + } + if result := run("--llm-extra-params={}"); result.Kind != commandkit.OutcomeSuccess || received.execution.Execution == nil || received.execution.Execution.ExtraParams == nil || len(received.execution.Execution.ExtraParams) != 0 { + t.Fatalf("empty object: %#v %#v", result, received) + } + for _, raw := range []string{"null", "[]", "1", "{invalid}"} { + before := calls + if result := run("--llm-extra-params=" + raw); result.Kind != commandkit.OutcomeUsage || calls != before { + t.Fatalf("invalid object %q: %#v", raw, result) + } + } + if result := run("--llm-backend=unused"); result.Kind != commandkit.OutcomeUsage { + t.Fatalf("unselected flag: %#v", result) + } +} + +func TestValidationForwardsExecutionControls(t *testing.T) { + var received promptkit.RunRequest + suite := ckprompt.Suite[struct{}, struct{}]{ValidateRequest: func(_ context.Context, req ckprompt.ValidationRequest, _, _ struct{}) (commandkit.Acquired[struct{}], error) { + received = req.Request + return commandkit.Acquired[struct{}]{}, nil + }} + app, err := commandkit.New(commandkit.Identity{Name: "app"}, ckprompt.Command(suite)) + if err != nil { + t.Fatal(err) + } + result := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile=", "--llm-max-tokens=0", "--llm-extra-params={}", "--llm-transient-retries=0"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if result.Kind != commandkit.OutcomeSuccess || received.Execution == nil || received.Execution.MaxTokens == nil || *received.Execution.MaxTokens != 0 || received.Execution.ExtraParams == nil || received.TransientRetries == nil || *received.TransientRetries != 0 { + t.Fatalf("result/request = %#v %#v", result, received) + } +} diff --git a/integrations/ckprompt/requirements_command_test.go b/integrations/ckprompt/requirements_command_test.go index 66e07eb..ebbc88f 100644 --- a/integrations/ckprompt/requirements_command_test.go +++ b/integrations/ckprompt/requirements_command_test.go @@ -108,17 +108,17 @@ func TestRequirementsCommandForwardsNativeRequestAndProjectsInspection(t *testin { name: "conditional discovery", request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"}, - args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial"}, }, { name: "supplied branch variable", request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", Vars: map[string]string{"mode": "full"}}, - args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--var", "mode=full", "--format", "yaml"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial", "--llm-var", "mode=full", "--format", "yaml"}, }, { name: "prompt with a required variable", request: promptkit.RequirementInspectionRequest{PromptID: "required-variable", ProfileID: "editorial"}, - args: []string{"promptkit", "requirements", "show", "required-variable", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "required-variable", "--llm-profile", "editorial"}, }, { name: "default profile selection", @@ -128,22 +128,22 @@ func TestRequirementsCommandForwardsNativeRequestAndProjectsInspection(t *testin { name: "first distinct prompt ID", request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-first", ProfileID: "editorial"}, - args: []string{"promptkit", "requirements", "show", "ambiguous-first", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "ambiguous-first", "--llm-profile", "editorial"}, }, { name: "second distinct prompt ID", request: promptkit.RequirementInspectionRequest{PromptID: "ambiguous-second", ProfileID: "editorial"}, - args: []string{"promptkit", "requirements", "show", "ambiguous-second", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "ambiguous-second", "--llm-profile", "editorial"}, }, { name: "session template context", request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial"}, - args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial"}, }, { name: "session ID context", request: promptkit.RequirementInspectionRequest{PromptID: "briefing", ProfileID: "editorial", SessionID: "session-42"}, - args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "editorial", "--session-id", "session-42"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "editorial", "--llm-session-id", "session-42"}, }, } { t.Run(test.name, func(t *testing.T) { @@ -164,17 +164,17 @@ func TestRequirementsCommandPreservesNativeSelectionErrors(t *testing.T) { }{ { name: "missing prompt", - args: []string{"promptkit", "requirements", "show", "missing", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "missing", "--llm-profile", "editorial"}, cause: promptkit.ErrPromptNotFound, }, { name: "missing profile", - args: []string{"promptkit", "requirements", "show", "briefing", "--profile", "missing"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--llm-profile", "missing"}, cause: promptkit.ErrProfileNotFound, }, { name: "removed prompt version flag", - args: []string{"promptkit", "requirements", "show", "briefing", "--version", "2", "--profile", "editorial"}, + args: []string{"promptkit", "requirements", "show", "briefing", "--version", "2", "--llm-profile", "editorial"}, cause: nil, }, } { diff --git a/integrations/ckprompt/smoke_aggregate_test.go b/integrations/ckprompt/smoke_aggregate_test.go index b0741d4..8bb4e4a 100644 --- a/integrations/ckprompt/smoke_aggregate_test.go +++ b/integrations/ckprompt/smoke_aggregate_test.go @@ -72,7 +72,7 @@ func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) app := newPromptTestApp(t, Command(suite)) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{ - "promptkit", "test", "all", "--model=base-model", "--temperature=0.25", "--transient-retries=3", + "promptkit", "test", "all", "--llm-model=base-model", "--llm-temperature=0.25", "--llm-transient-retries=3", }, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) { t.Fatalf("aggregate outcome = %#v", outcome) diff --git a/integrations/ckprompt/smoke_command.go b/integrations/ckprompt/smoke_command.go index 36ce8e2..6f3edd4 100644 --- a/integrations/ckprompt/smoke_command.go +++ b/integrations/ckprompt/smoke_command.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "io" - "strconv" "go.maximumdirect.net/ckconfig" commandkit "go.maximumdirect.net/commandkit" @@ -15,6 +14,8 @@ type smokeCommandInput[I any] struct { selection ckconfig.Selection kind string request promptkit.SmokeTestRequest + execution ExecutionInput + resources ResourceInput captured commandkit.FlagValues[I] selected I } @@ -35,98 +36,13 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command { func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).selection.Overlay = value }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "backend", Usage: "PromptKit backend ID."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - (*input).request.BackendID = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "prompt", Usage: "PromptKit prompt ID."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - (*input).request.PromptID = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "PromptKit profile ID."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - (*input).request.ProfileID = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "endpoint", Usage: "Execution endpoint override."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.Endpoint = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "model", Usage: "Execution model override."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.Model = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "temperature", Usage: "Execution temperature override."}, (*float64)(nil), parseSmokeFloat, - func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.Temperature = &value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "max-tokens", Usage: "Execution maximum tokens override."}, (*int)(nil), strconv.Atoi, - func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.MaxTokens = &value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "top-p", Usage: "Execution top-p override."}, (*float64)(nil), parseSmokeFloat, - func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.TopP = &value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "timeout-seconds", Usage: "Execution timeout override."}, (*int)(nil), strconv.Atoi, - func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.TimeoutSeconds = &value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "service-tier", Usage: "Execution service tier."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.ServiceTier = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "reasoning-effort", Usage: "Execution reasoning effort override."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.ReasoningEffort = &value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "api-key-env", Usage: "Execution API key environment variable."}, (*string)(nil), parseDefinitionString, - func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { - if !value.Set { - return - } - execution := smokeExecution(input) - execution.APIKeyEnv = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "transient-retries", Usage: "Inference transient retry count."}, (*int)(nil), strconv.Atoi, - func(input **smokeCommandInput[I], value commandkit.Supplied[int]) { - if value.Set { - (*input).request.TransientRetries = &value.Value - } - }), ) + resources := NewResourceFlags(func(input **smokeCommandInput[I]) *ResourceInput { return &(*input).resources }) + execution := NewExecutionFlags(func(input **smokeCommandInput[I]) *ExecutionInput { return &(*input).execution }) + flags = append(flags, resources.Backend, resources.Prompt, resources.Profile) + flags = append(flags, execution.All()...) + return commandkit.NewCommand(commandkit.CommandSpec[*smokeCommandInput[I]]{ Name: "test", Summary: "Run one PromptKit smoke test.", @@ -142,6 +58,11 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command { if input == nil { return errors.New("PromptKit smoke-test input must not be nil") } + input.request.BackendID = input.resources.Backend.Value + input.request.PromptID = input.resources.Prompt.Value + input.request.ProfileID = input.resources.Profile.Value + input.request.Execution = input.execution.Execution + input.request.TransientRetries = input.execution.TransientRetries if input.kind != "all" { input.request.Kind = promptkit.SmokeTestKind(input.kind) } @@ -158,17 +79,6 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command { }) } -func smokeExecution[I any](input **smokeCommandInput[I]) *promptkit.ExecutionTargetOverride { - if (*input).request.Execution == nil { - (*input).request.Execution = &promptkit.ExecutionTargetOverride{} - } - return (*input).request.Execution -} - -func parseSmokeFloat(value string) (float64, error) { - return strconv.ParseFloat(value, 64) -} - func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome { if err := promptKitContextError(invocation.Context); err != nil { return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err) diff --git a/integrations/ckprompt/smoke_command_test.go b/integrations/ckprompt/smoke_command_test.go index 3ef2ed3..062f9e3 100644 --- a/integrations/ckprompt/smoke_command_test.go +++ b/integrations/ckprompt/smoke_command_test.go @@ -53,9 +53,9 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) { var output bytes.Buffer outcome = app.Execute(context.Background(), []string{ "promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring", - "--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid", - "--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", - "--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0", + "--llm-backend=local", "--llm-prompt=briefing", "--llm-profile=editorial", "--llm-endpoint=https://example.invalid", + "--llm-model=fixture", "--llm-temperature=0", "--llm-max-tokens=0", "--llm-top-p=0", "--llm-timeout-seconds=0", + "--llm-service-tier=", "--llm-reasoning-effort=", "--llm-api-key-env=TOKEN", "--llm-transient-retries=0", }, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 { t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) diff --git a/integrations/ckprompt/smoke_lifecycle_test.go b/integrations/ckprompt/smoke_lifecycle_test.go index 959cc79..c05e33f 100644 --- a/integrations/ckprompt/smoke_lifecycle_test.go +++ b/integrations/ckprompt/smoke_lifecycle_test.go @@ -37,9 +37,9 @@ func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T first := app.Execute(context.Background(), []string{ "promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml", "--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final", - "--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt", - "--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model", - "--temperature=0", "--transient-retries=0", + "--llm-backend=old-backend", "--llm-backend=final-backend", "--llm-prompt=old-prompt", "--llm-prompt=final-prompt", + "--llm-profile=old-profile", "--llm-profile=final-profile", "--llm-model=old-model", "--llm-model=final-model", + "--llm-temperature=0", "--llm-transient-retries=0", }, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 { t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests) @@ -73,11 +73,11 @@ func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) { }{ {name: "missing kind", args: []string{"promptkit", "test"}}, {name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}}, - {name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}}, - {name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}}, - {name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}}, - {name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}}, - {name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}}, + {name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--llm-temperature=warm"}}, + {name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--llm-max-tokens=many"}}, + {name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--llm-top-p=high"}}, + {name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--llm-timeout-seconds=soon"}}, + {name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--llm-transient-retries=several"}}, {name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=invalid"}}, {name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}}, } { diff --git a/integrations/ckprompt/validation_adapters_test.go b/integrations/ckprompt/validation_adapters_test.go index a157ed3..7fbc12e 100644 --- a/integrations/ckprompt/validation_adapters_test.go +++ b/integrations/ckprompt/validation_adapters_test.go @@ -3,6 +3,7 @@ package ckprompt import ( "context" "errors" + "io" "strings" "testing" "testing/fstest" @@ -331,3 +332,25 @@ func TestWithRequestValidationCancellationBoundaries(t *testing.T) { } }) } + +func TestValidationBuilderReplacesCLIExecutionControls(t *testing.T) { + engine := adapterEngine(t) + calls := 0 + suite := WithRequestValidation(Suite[struct{}, struct{}]{}, + func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return engine, nil + }, + func(_ context.Context, seed ValidationRequest, _, _ struct{}) (promptkit.RunRequest, error) { + calls++ + if seed.Request.Execution == nil || seed.Request.Execution.MaxTokens == nil || *seed.Request.Execution.MaxTokens != -1 { + t.Fatalf("builder seed = %#v", seed.Request) + } + return promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}, nil + }, + ) + app := newPromptTestApp(t, Command(suite)) + result := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-max-tokens=-1"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if result.Kind != commandkit.OutcomeSuccess || calls != 1 { + t.Fatalf("builder replacement = %#v, calls = %d", result, calls) + } +} diff --git a/integrations/ckprompt/validation_command.go b/integrations/ckprompt/validation_command.go index 903efe6..3fa396b 100644 --- a/integrations/ckprompt/validation_command.go +++ b/integrations/ckprompt/validation_command.go @@ -5,14 +5,16 @@ import ( "fmt" commandkit "go.maximumdirect.net/commandkit" - promptkit "go.maximumdirect.net/promptkit" ) type validationCommandInput[I any] struct { - promptID string - request ValidationRequest - captured commandkit.FlagValues[I] - selected I + promptID string + request ValidationRequest + execution ExecutionInput + resources ResourceInput + inputs PromptInputs + captured commandkit.FlagValues[I] + selected I } type requestValidationReport struct { @@ -36,32 +38,14 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command { func(input **validationCommandInput[I], value commandkit.Supplied[string]) { (*input).request.Selection.Overlay = value }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "Profile identity for validation."}, stringDefault(""), parseDefinitionString, - func(input **validationCommandInput[I], value commandkit.Supplied[string]) { - (*input).request.Request.ProfileID = value.Value - }), - commandkit.ValueFlag(commandkit.FlagMeta{Name: "session-id", Usage: "Session context for validation."}, stringDefault(""), parseDefinitionString, - func(input **validationCommandInput[I], value commandkit.Supplied[string]) { - (*input).request.Request.SessionID = value.Value - }), - commandkit.RepeatFlag(commandkit.FlagMeta{Name: "var", Usage: "Caller variable assignment KEY=VALUE."}, []variableAssignment(nil), parseVariableAssignment, - func(input **validationCommandInput[I], value commandkit.Supplied[[]variableAssignment]) { - variables := make(map[string]string, len(value.Value)) - for _, assignment := range value.Value { - variables[assignment.key] = assignment.value - } - (*input).request.Request.Vars = variables - }), - commandkit.RepeatFlag(commandkit.FlagMeta{Name: "input", Usage: "Prompt input file assignment NAME=PATH."}, []inputAssignment(nil), parseInputAssignment, - func(input **validationCommandInput[I], value commandkit.Supplied[[]inputAssignment]) { - inputs := make(map[string]promptkit.ArtifactRef, len(value.Value)) - for _, assignment := range value.Value { - inputs[assignment.name] = promptkit.File(assignment.path) - } - (*input).request.Request.Inputs = inputs - }), ) + resources := NewResourceFlags(func(input **validationCommandInput[I]) *ResourceInput { return &(*input).resources }) + execution := NewExecutionFlags(func(input **validationCommandInput[I]) *ExecutionInput { return &(*input).execution }) + inputs := NewPromptInputFlags(func(input **validationCommandInput[I]) *PromptInputs { return &(*input).inputs }) + flags = append(flags, resources.Profile, resources.SessionID, inputs.Variable, inputs.Text, inputs.File) + flags = append(flags, execution.All()...) + return commandkit.NewCommand(commandkit.CommandSpec[*validationCommandInput[I]]{ Name: "validate", Summary: "Validate a concrete PromptKit request without executing it.", @@ -80,6 +64,12 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command { if input == nil { return errors.New("PromptKit validation input must not be nil") } + input.request.Request.ProfileID = input.resources.Profile.Value + input.request.Request.SessionID = input.resources.SessionID.Value + input.request.Request.Vars = input.inputs.Vars + input.request.Request.Inputs = input.inputs.Inputs + input.request.Request.Execution = input.execution.Execution + input.request.Request.TransientRetries = input.execution.TransientRetries selected, err := prepareSuiteInput(suite, input.request.Selection, input.captured) if err != nil { return err diff --git a/integrations/ckprompt/validation_command_test.go b/integrations/ckprompt/validation_command_test.go index 2ad2daa..8567162 100644 --- a/integrations/ckprompt/validation_command_test.go +++ b/integrations/ckprompt/validation_command_test.go @@ -64,8 +64,8 @@ func TestValidationCommandComposesWithRequirementsShow(t *testing.T) { outcome = app.Execute(context.Background(), []string{ "promptkit", "requirements", "validate", "briefing", "--config", "config.yml", "--overlay", "overlay.yml", "--campaign", "spring", - "--profile", "editorial", "--session-id", "session-1", - "--var", "topic=roadmap", "--input", "notes=fixtures/notes=final.txt", "--format", "yaml", + "--llm-profile", "editorial", "--llm-session-id", "session-1", + "--llm-var", "topic=roadmap", "--llm-input-file", "notes=fixtures/notes=final.txt", "--format", "yaml", }, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || validateCalls != 1 { t.Fatalf("validation outcome = (%#v, %q), calls=%d", outcome, output.String(), validateCalls) @@ -168,7 +168,7 @@ func TestValidationCommandConstructionAndCaptureLifecycle(t *testing.T) { } var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-profile", "default", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || newInputCalls != 1 || newStateCalls != 1 || checkCalls != 1 || buildCalls != 1 || loadCalls != 1 { t.Fatalf("captured command = %#v, callback counts input/state/check/build/load=%d/%d/%d/%d/%d", outcome, newInputCalls, newStateCalls, checkCalls, buildCalls, loadCalls) } @@ -197,9 +197,9 @@ func TestValidationCommandPreservesPresenceAndRepeatedAssignmentsPerInvocation(t firstArgs := []string{ "promptkit", "requirements", "validate", "briefing", "--config", "", "--overlay", "", - "--profile", "", "--session-id", "", "--campaign", "spring", - "--var", "=", "--var", "phrase= two words, three ", "--var", "duplicate=first", "--var", "duplicate=last=value", - "--input", "=", "--input", "file= path, with commas=and=equals ", "--input", "duplicate=first.txt", "--input", "duplicate=last=path.txt", + "--llm-profile", "", "--llm-session-id", "", "--campaign", "spring", + "--llm-var", "=", "--llm-var", "phrase= two words, three ", "--llm-var", "duplicate=first", "--llm-var", "duplicate=last=value", + "--llm-input-file", "=", "--llm-input-file", "file= path, with commas=and=equals ", "--llm-input-file", "duplicate=first.txt", "--llm-input-file", "duplicate=last=path.txt", } outcome := app.Execute(context.Background(), firstArgs, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || len(requests) != 1 { @@ -257,8 +257,8 @@ func TestValidationCommandUsageAndInputFailuresStopProviderWork(t *testing.T) { {name: "unsupported format", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--format", "invalid"}}, {name: "unknown flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--unknown", "value"}}, {name: "removed prompt version flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--version", "2"}}, - {name: "malformed variable before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--var", "broken", "--var", "mode=full"}}, - {name: "malformed input before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "broken", "--input", "notes=file.txt"}}, + {name: "malformed variable before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-var", "broken", "--llm-var", "mode=full"}}, + {name: "malformed input before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--llm-input-file", "broken", "--llm-input-file", "notes=file.txt"}}, {name: "missing required selector", args: []string{"promptkit", "requirements", "validate", "briefing"}}, } { t.Run(test.name, func(t *testing.T) { @@ -325,7 +325,7 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) app := newPromptTestApp(t, Command(suite)) var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--llm-profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 { t.Fatalf("native request validation = %#v, output=%q", outcome, output.String()) } @@ -342,8 +342,8 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) args []string want error }{ - {name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender}, - {name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound}, + {name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile", "default"}, want: promptkit.ErrPromptRender}, + {name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--llm-profile", "missing", "--llm-input-file", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound}, } { t.Run(test.name, func(t *testing.T) { output.Reset()