From f7174a078e7cb8b16f4d16253f08dce09fa083ab Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 11 Oct 2026 01:46:54 +0000 Subject: [PATCH] Add PromptKit inspection commands --- docs/roadmap/implementation.md | 15 +- integrations/ckprompt/adapters.go | 7 + integrations/ckprompt/adapters_test.go | 31 +++ integrations/ckprompt/command.go | 357 +++++++++++++++++++++++++ integrations/ckprompt/command_test.go | 311 +++++++++++++++++++++ 5 files changed, 720 insertions(+), 1 deletion(-) create mode 100644 integrations/ckprompt/command.go create mode 100644 integrations/ckprompt/command_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 59f4fd0..8333c42 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implement the optional integration modules and captured registration API -Status: implementation in progress. Stages 1 through 11 are complete; subsequent work follows the ordered roadmap. +Status: implementation in progress. Stages 1 through 12 are complete; subsequent work follows the ordered roadmap. Execute the stages in order. Each stage is sized as a separate Luna coding-agent assignment. This document authorizes no publication or changes to sibling repositories. @@ -590,6 +590,19 @@ adapter, unused malformed binding, actual installed namespace collisions, definition input/state lifetimes, native report fields, unknown/blank lookup, ExtraParams, and failed output. Run the complete ckprompt suite when coherent. +**Completion evidence — complete:** `ckprompt.Command` builds a root-only, +capability-driven `promptkit show` tree, ignores selectors for empty suites, and +retains empty-adapter construction issues. Generated commands use captured core +flags, required ID operands, config/overlay selection, prompt-only version, and +validated text/YAML formatting. Tests cover installed namespace conflicts, +selected input/state lifetimes, missing versus blank IDs, native lookup causes, +diagnostics, cancellation and report delivery. Native prompt/profile reports +preserve field order, zero values, empty lists, opaque hashes and nested +`ExtraParams`; profile output reports environment-variable names and requirement +status without resolving credentials. Focused ckprompt adapter/command tests and +root tests pass. The complete ckprompt suite remains deferred until Stage 13 +replaces the prototype sources and mixed-package tests. + ## Stage 13: Finish maintained consumers and module boundary evidence **Outcome:** all three adoption paths use the final public API with appropriate diff --git a/integrations/ckprompt/adapters.go b/integrations/ckprompt/adapters.go index 990fca2..6db7545 100644 --- a/integrations/ckprompt/adapters.go +++ b/integrations/ckprompt/adapters.go @@ -143,6 +143,13 @@ func promptKitDiagnostics(scope string, sources []configkit.SourceInfo, records // Snapshot provider-owned slices before deferring their report to invocation. sources = append([]configkit.SourceInfo(nil), sources...) records = append([]configkit.Diagnostic(nil), records...) + for index := range records { + if records[index].Primary != nil { + primary := *records[index].Primary + records[index].Primary = &primary + } + records[index].Related = append([]configkit.Location(nil), records[index].Related...) + } return []commandkit.Diagnostic{{ Severity: severity, Write: func(writer io.Writer) error { diff --git a/integrations/ckprompt/adapters_test.go b/integrations/ckprompt/adapters_test.go index 200b2b8..5d2fb64 100644 --- a/integrations/ckprompt/adapters_test.go +++ b/integrations/ckprompt/adapters_test.go @@ -18,12 +18,30 @@ import ( const adapterPrompt = `prompts: briefing: "1": + default_profile: default + inputs: + - name: topic + required: true + content_type: text/plain + description: "Topic" + - name: audience + required: false + content_type: text/plain + description: "" messages: - role: user content: "Hello" output: format: text validation_mode: none + empty: + "1": + messages: + - role: user + content: "Empty inputs" + output: + format: text + validation_mode: none backends: local: endpoint: "https://example.invalid/v1" @@ -31,6 +49,19 @@ profiles: default: backend: local model: example + temperature: 0 + max_tokens: 0 + api_key_env: PROMPTKIT_ADAPTER_TEST_KEY + extra_params: + token: "caller-owned-value" + nested: + visible: true + null_value: null + sequence: [first, null, 3] + requires-key: + backend: local + model: example + api_key_required: true ` func adapterEngine(t *testing.T) *promptkit.Engine { diff --git a/integrations/ckprompt/command.go b/integrations/ckprompt/command.go new file mode 100644 index 0000000..0e95a8b --- /dev/null +++ b/integrations/ckprompt/command.go @@ -0,0 +1,357 @@ +package ckprompt + +import ( + "bytes" + "errors" + "fmt" + "io" + "strconv" + "strings" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" + "go.yaml.in/yaml/v4" +) + +type definitionInput[I any] struct { + selection ckconfig.Selection + id string + version commandkit.Supplied[string] + format string + captured commandkit.FlagValues[I] + selected I +} + +// Command returns the optional named PromptKit inspection tree. It adds no +// configuration family; callers compose configuration scopes independently. +func Command[I, S any](suite Suite[I, S]) commandkit.Command { + if suite.InspectPrompt == nil && suite.InspectProfile == nil { + command := commandkit.NoCommand() + for _, issue := range suite.issues { + command = commandkit.WithError(command, issue) + } + return command + } + + showCommands := make([]commandkit.Command, 0, 2) + if suite.InspectPrompt != nil { + showCommands = append(showCommands, newDefinitionCommand(suite, "prompt", true)) + } + if suite.InspectProfile != nil { + showCommands = append(showCommands, newDefinitionCommand(suite, "profile", false)) + } + command := commandkit.RootGroup("promptkit", "Inspect named PromptKit definitions.", + commandkit.Group("show", "Show a named PromptKit definition.", showCommands...), + ) + for _, issue := range suite.issues { + command = commandkit.WithError(command, issue) + } + return command +} + +func newDefinitionCommand[I, S any](suite Suite[I, S], name string, prompt bool) commandkit.Command { + flags := commandkit.CaptureFlags(suite.Selectors, func(input **definitionInput[I]) *commandkit.FlagValues[I] { + if input == nil || *input == nil { + return nil + } + return &(*input).captured + }) + flags = append(flags, + commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Configuration input path."}, (*string)(nil), parseDefinitionString, + func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).selection.Config = value }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay path."}, (*string)(nil), parseDefinitionString, + func(input **definitionInput[I], value commandkit.Supplied[string]) { + (*input).selection.Overlay = value + }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "format", Usage: "Report format: text or yaml."}, stringDefault("text"), parseDefinitionString, + func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).format = value.Value }), + ) + if prompt { + flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: "version", Usage: "Prompt version to inspect."}, (*string)(nil), parseDefinitionString, + func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).version = value })) + } + + spec := commandkit.CommandSpec[*definitionInput[I]]{ + Name: name, + Summary: "Inspect a named PromptKit " + name + ".", + NewInput: func() *definitionInput[I] { + return &definitionInput[I]{} + }, + Flags: flags, + Arguments: []commandkit.Argument[*definitionInput[I]]{ + commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "id", Usage: "Prompt or profile ID."}, true, parseDefinitionString, + func(input **definitionInput[I], value commandkit.Supplied[string]) { (*input).id = value.Value }), + }, + CheckInput: func(input *definitionInput[I]) error { + if err := checkDefinitionFormat(input.format); err != nil { + return err + } + selected, err := prepareDefinitionInput(suite, input) + if err != nil { + return err + } + input.selected = selected + return nil + }, + Run: func(invocation commandkit.Invocation, input *definitionInput[I]) commandkit.Outcome { + return runDefinitionInspection(invocation, suite, input, prompt) + }, + } + return commandkit.NewCommand(spec) +} + +func prepareDefinitionInput[I, S any](suite Suite[I, S], input *definitionInput[I]) (I, error) { + var selected I + if suite.NewInput != nil { + selected = suite.NewInput() + } + if err := input.captured.ApplyTo(&selected, suite.Selectors...); err != nil { + return selected, err + } + if suite.CheckInput != nil { + if err := suite.CheckInput(input.selection, selected); err != nil { + return selected, err + } + } + return selected, nil +} + +func runDefinitionInspection[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *definitionInput[I], prompt bool) commandkit.Outcome { + if invocation.Context != nil { + if err := invocation.Context.Err(); err != nil { + return definitionFailure(err, nil) + } + } + var state S + if suite.NewState != nil { + state = suite.NewState() + } + if invocation.Context != nil { + if err := invocation.Context.Err(); err != nil { + return definitionFailure(err, nil) + } + } + + if prompt { + request := PromptRequest{Selection: input.selection, ID: input.id, Version: input.version} + acquired, err := suite.InspectPrompt(invocation.Context, request, input.selected, state) + if err != nil { + return definitionFailure(err, acquired.Diagnostics) + } + if acquired.Value == nil { + return definitionFailure(errors.New("PromptKit prompt inspection returned nil"), acquired.Diagnostics) + } + if invocation.Context != nil { + if err := invocation.Context.Err(); err != nil { + return definitionFailure(err, acquired.Diagnostics) + } + } + return renderPromptReport(invocation, promptReportFromInspection(acquired.Value), input.format, acquired.Diagnostics) + } + + request := ProfileRequest{Selection: input.selection, ID: input.id} + acquired, err := suite.InspectProfile(invocation.Context, request, input.selected, state) + if err != nil { + return definitionFailure(err, acquired.Diagnostics) + } + if acquired.Value == nil { + return definitionFailure(errors.New("PromptKit profile inspection returned nil"), acquired.Diagnostics) + } + if invocation.Context != nil { + if err := invocation.Context.Err(); err != nil { + return definitionFailure(err, acquired.Diagnostics) + } + } + return renderProfileReport(invocation, profileReportFromInspection(acquired.Value), input.format, acquired.Diagnostics) +} + +type promptInputReport struct { + Name string `yaml:"name"` + Required bool `yaml:"required"` + ContentType string `yaml:"content_type"` + Description string `yaml:"description"` +} + +type promptOutputContractReport struct { + Format string `yaml:"format"` + ValidationMode string `yaml:"validation_mode"` + SchemaPath string `yaml:"schema_path"` + RepairAttempts int `yaml:"repair_attempts"` +} + +type promptInspectionReport struct { + PromptID string `yaml:"prompt_id"` + PromptVersion string `yaml:"prompt_version"` + PromptHash string `yaml:"prompt_hash"` + DefaultProfileID string `yaml:"default_profile_id"` + Inputs []promptInputReport `yaml:"inputs"` + OutputContract promptOutputContractReport `yaml:"output_contract"` +} + +type modelParamsReport struct { + BackendID string `yaml:"backend_id"` + Endpoint string `yaml:"endpoint"` + Model string `yaml:"model"` + Temperature float64 `yaml:"temperature"` + MaxTokens int `yaml:"max_tokens"` + TopP float64 `yaml:"top_p"` + TimeoutSeconds int `yaml:"timeout_seconds"` + ServiceTier string `yaml:"service_tier"` + ReasoningEffort string `yaml:"reasoning_effort"` + APIKeyEnv string `yaml:"api_key_env"` + ExtraParams map[string]any `yaml:"extra_params"` +} + +type profileInspectionReport struct { + ProfileID string `yaml:"profile_id"` + EffectiveModelParams modelParamsReport `yaml:"effective_model_params"` + APIKeyRequired bool `yaml:"api_key_required"` +} + +func promptReportFromInspection(inspection *promptkit.PromptInspection) promptInspectionReport { + report := promptInspectionReport{ + PromptID: inspection.PromptID, + PromptVersion: inspection.PromptVersion, + PromptHash: inspection.PromptHash, + DefaultProfileID: inspection.DefaultProfileID, + Inputs: make([]promptInputReport, 0, len(inspection.Inputs)), + OutputContract: promptOutputContractReport{ + Format: string(inspection.OutputContract.Format), + ValidationMode: string(inspection.OutputContract.ValidationMode), + SchemaPath: inspection.OutputContract.SchemaPath, + RepairAttempts: inspection.OutputContract.RepairAttempts, + }, + } + for _, input := range inspection.Inputs { + report.Inputs = append(report.Inputs, promptInputReport{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}) + } + return report +} + +func profileReportFromInspection(inspection *promptkit.ProfileInspection) profileInspectionReport { + params := inspection.EffectiveModelParams + extra := params.ExtraParams + if extra == nil { + extra = map[string]any{} + } + return profileInspectionReport{ + ProfileID: inspection.ProfileID, + EffectiveModelParams: modelParamsReport{ + BackendID: params.BackendID, Endpoint: params.Endpoint, Model: params.Model, + Temperature: params.Temperature, MaxTokens: params.MaxTokens, TopP: params.TopP, + TimeoutSeconds: params.TimeoutSeconds, ServiceTier: params.ServiceTier, + ReasoningEffort: params.ReasoningEffort, APIKeyEnv: params.APIKeyEnv, ExtraParams: extra, + }, + APIKeyRequired: inspection.APIKeyRequired, + } +} + +func renderPromptReport(invocation commandkit.Invocation, report promptInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome { + var output bytes.Buffer + if format == "yaml" { + if err := encodePromptYAML(&output, report); err != nil { + return definitionFailure(err, diagnostics) + } + } else { + output.WriteString(formatPromptText(report)) + } + return writePromptReport(invocation, output.Bytes(), diagnostics) +} + +func renderProfileReport(invocation commandkit.Invocation, report profileInspectionReport, format string, diagnostics []commandkit.Diagnostic) commandkit.Outcome { + var output bytes.Buffer + if format == "yaml" { + if err := encodePromptYAML(&output, report); err != nil { + return definitionFailure(err, diagnostics) + } + } else { + text, err := formatProfileText(report) + if err != nil { + return definitionFailure(err, diagnostics) + } + output.WriteString(text) + } + return writePromptReport(invocation, output.Bytes(), diagnostics) +} + +func formatPromptText(report promptInspectionReport) string { + var output strings.Builder + fmt.Fprintf(&output, "Prompt ID: %s\n", strconv.Quote(report.PromptID)) + fmt.Fprintf(&output, "Prompt version: %s\n", strconv.Quote(report.PromptVersion)) + fmt.Fprintf(&output, "Prompt hash: %s\n", strconv.Quote(report.PromptHash)) + fmt.Fprintf(&output, "Default profile ID: %s\n", strconv.Quote(report.DefaultProfileID)) + output.WriteString("Inputs:\n") + for _, input := range report.Inputs { + fmt.Fprintf(&output, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", strconv.Quote(input.Name), input.Required, strconv.Quote(input.ContentType), strconv.Quote(input.Description)) + } + if len(report.Inputs) == 0 { + output.WriteString(" []\n") + } + fmt.Fprintf(&output, "Output contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", strconv.Quote(report.OutputContract.Format), strconv.Quote(report.OutputContract.ValidationMode), strconv.Quote(report.OutputContract.SchemaPath), report.OutputContract.RepairAttempts) + return output.String() +} + +func formatProfileText(report profileInspectionReport) (string, error) { + params := report.EffectiveModelParams + var output strings.Builder + fmt.Fprintf(&output, "Profile ID: %s\n", strconv.Quote(report.ProfileID)) + output.WriteString("Effective model params:\n") + fmt.Fprintf(&output, " backend_id: %s\n endpoint: %s\n model: %s\n temperature: %s\n max_tokens: %d\n top_p: %s\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n", strconv.Quote(params.BackendID), strconv.Quote(params.Endpoint), strconv.Quote(params.Model), strconv.FormatFloat(params.Temperature, 'g', -1, 64), params.MaxTokens, strconv.FormatFloat(params.TopP, 'g', -1, 64), params.TimeoutSeconds, strconv.Quote(params.ServiceTier), strconv.Quote(params.ReasoningEffort), strconv.Quote(params.APIKeyEnv)) + output.WriteString(" extra_params:\n") + var nested strings.Builder + if err := encodePromptYAML(&nested, params.ExtraParams); err != nil { + return "", err + } + for _, line := range strings.Split(strings.TrimSuffix(nested.String(), "\n"), "\n") { + output.WriteString(" ") + output.WriteString(line) + output.WriteByte('\n') + } + fmt.Fprintf(&output, "API key required: %t\n", report.APIKeyRequired) + return output.String(), nil +} + +func encodePromptYAML(writer io.Writer, value any) error { + encoder := yaml.NewEncoder(writer) + encoder.SetIndent(2) + return encoder.Encode(value) +} + +func writePromptReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic) commandkit.Outcome { + if invocation.Context != nil { + if err := invocation.Context.Err(); err != nil { + return definitionFailure(err, diagnostics) + } + } + written, err := invocation.Streams.Out.Write(output) + if err == nil && written != len(output) { + err = io.ErrShortWrite + } + if err != nil { + outcome := definitionFailure(err, diagnostics) + outcome.ReportIncomplete = true + outcome.DeliveryError = err + return outcome + } + outcome := commandkit.Success() + outcome.Diagnostics = diagnostics + return outcome +} + +func definitionFailure(cause error, diagnostics []commandkit.Diagnostic) commandkit.Outcome { + outcome := commandkit.Failure(cause) + outcome.Diagnostics = diagnostics + return outcome +} + +func checkDefinitionFormat(format string) error { + if format != "text" && format != "yaml" { + return fmt.Errorf("unsupported format %q; expected text or yaml", format) + } + return nil +} + +func parseDefinitionString(value string) (string, error) { return value, nil } + +func stringDefault(value string) *string { return &value } diff --git a/integrations/ckprompt/command_test.go b/integrations/ckprompt/command_test.go new file mode 100644 index 0000000..92499e8 --- /dev/null +++ b/integrations/ckprompt/command_test.go @@ -0,0 +1,311 @@ +package ckprompt + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + + commandkit "gitea.maximumdirect.net/eric/CommandKit" + "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + promptkit "gitea.maximumdirect.net/eric/promptkit" + "go.yaml.in/yaml/v4" +) + +func newPromptTestApp(t *testing.T, commands ...commandkit.Command) *commandkit.App { + t.Helper() + app, err := commandkit.New(commandkit.Identity{Name: "example"}, commands...) + if err != nil { + t.Fatal(err) + } + return app +} + +func TestCommandBuildsIndependentOptionalTrees(t *testing.T) { + promptOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { + return commandkit.Acquired[*promptkit.PromptInspection]{}, errors.New("unused") + }})) + var output bytes.Buffer + outcome := promptOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n prompt ") || strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "config") { + t.Fatalf("prompt-only help = (%#v, %q)", outcome, output.String()) + } + + profileOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { + return commandkit.Acquired[*promptkit.ProfileInspection]{}, errors.New("unused") + }})) + output.Reset() + outcome = profileOnly.Execute(context.Background(), []string{"help", "promptkit", "show"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "\n profile ") || strings.Contains(output.String(), "\n prompt ") { + t.Fatalf("profile-only help = (%#v, %q)", outcome, output.String()) + } + + both := Command(Suite[struct{}, struct{}]{ + InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { + return commandkit.Acquired[*promptkit.PromptInspection]{}, nil + }, + InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { + return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil + }, + }) + if err := newPromptTestApp(t).Register(both); err != nil { + t.Fatalf("both-capability tree failed to register: %v", err) + } + + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}})) + output.Reset() + outcome = app.Execute(context.Background(), []string{"help"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(output.String(), "promptkit") { + t.Fatalf("empty suite installed a namespace: (%#v, %q)", outcome, output.String()) + } + + invalid := FromEngine[int, int](nil) + invalid.InspectPrompt, invalid.InspectProfile = nil, nil + if err := newPromptTestApp(t).Register(Command(invalid)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") { + t.Fatalf("empty invalid adapter lost its construction issue: %v", err) + } +} + +func TestCommandUsesInstalledNamespaceCollisionChecks(t *testing.T) { + command := Command(Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { + return commandkit.Acquired[*promptkit.PromptInspection]{}, nil + }}) + other := commandkit.Group("promptkit", "Another command tree.", commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "child", Summary: "A child."})) + first, err := commandkit.New(commandkit.Identity{Name: "example"}, other) + if err != nil { + t.Fatal(err) + } + if err := first.Register(command); err == nil { + t.Fatal("PromptKit tree installed over an existing namespace") + } + second, err := commandkit.New(commandkit.Identity{Name: "example"}, command) + if err != nil { + t.Fatal(err) + } + if err := second.Register(other); err == nil { + t.Fatal("later command tree installed over the PromptKit namespace") + } +} + +func TestCommandValidatesOnlyGeneratedCapturedBindings(t *testing.T) { + unused := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}}) + if err := newPromptTestApp(t).Register(unused); err != nil { + t.Fatalf("unused malformed selector affected empty suite: %v", err) + } + generated := Command(Suite[struct{}, struct{}]{Selectors: []commandkit.Flag[struct{}]{{}}, InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { + return commandkit.Acquired[*promptkit.ProfileInspection]{}, nil + }}) + if err := newPromptTestApp(t).Register(generated); err == nil { + t.Fatal("generated command accepted an invalid captured selector") + } +} + +func TestDefinitionInputAndStateLifetimesFollowChecks(t *testing.T) { + type inspectInput struct{ Campaign commandkit.Supplied[string] } + type runState struct{} + conversions := 0 + flag := commandkit.ValueFlag[*inspectInput, string](commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil), func(value string) (string, error) { + conversions++ + return value, nil + }, func(input **inspectInput, value commandkit.Supplied[string]) { (*input).Campaign = value }) + newInputs, newStates, checks, callbacks := 0, 0, 0, 0 + rejectInput := false + var checked *inspectInput + var wantContext context.Context + suite := Suite[*inspectInput, *runState]{ + NewInput: func() *inspectInput { newInputs++; return &inspectInput{} }, + NewState: func() *runState { + newStates++ + if checks != 1 { + t.Errorf("state initialized after %d input checks", checks) + } + return &runState{} + }, + Selectors: []commandkit.Flag[*inspectInput]{flag}, + CheckInput: func(selection ckconfig.Selection, input *inspectInput) error { + checks++ + checked = input + if rejectInput { + return errors.New("rejected definition input") + } + if !selection.Config.Set || selection.Config.Value != "base.yml" || !selection.Overlay.Set || selection.Overlay.Value != "local.yml" || !input.Campaign.Set || input.Campaign.Value != "spring" { + return errors.New("definition input was incomplete") + } + return nil + }, + InspectPrompt: func(ctx context.Context, request PromptRequest, input *inspectInput, state *runState) (commandkit.Acquired[*promptkit.PromptInspection], error) { + callbacks++ + if ctx != wantContext || input != checked || request.ID != "briefing" || !request.Version.Set || request.Version.Value != "2" || state == nil { + t.Errorf("prompt callback arguments = ctx=%v request=%#v input=%p state=%p", ctx, request, input, state) + } + return commandkit.Acquired[*promptkit.PromptInspection]{Value: &promptkit.PromptInspection{PromptID: request.ID, PromptVersion: request.Version.Value}}, nil + }, + } + app := newPromptTestApp(t, Command(suite)) + wantContext = context.WithValue(context.Background(), struct{}{}, "context") + var output bytes.Buffer + outcome := app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || newInputs != 1 || newStates != 1 || checks != 1 || callbacks != 1 || conversions != 1 { + t.Fatalf("execution outcome=%#v counts conversion/input/state/check/callback=%d/%d/%d/%d/%d report=%q", outcome, conversions, newInputs, newStates, checks, callbacks, output.String()) + } + rejectInput = true + output.Reset() + outcome = app.Execute(wantContext, []string{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--config", "base.yml", "--overlay", "local.yml", "--version", "2"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || newInputs != 2 || newStates != 1 || checks != 2 || callbacks != 1 || conversions != 2 { + t.Fatalf("rejected input reached state/callback: outcome=%#v counts=%d/%d/%d/%d/%d", outcome, conversions, newInputs, newStates, checks, callbacks) + } +} + +func TestDefinitionReportsRetainNativeFieldsAndSafeCredentialReferences(t *testing.T) { + t.Setenv("PROMPTKIT_ADAPTER_TEST_KEY", "credential-value-must-not-appear") + engine := adapterEngine(t) + loadCalls := 0 + suite := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loadCalls++ + return engine, nil + }) + app := newPromptTestApp(t, Command(suite)) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing", "--version", "1", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 1 { + t.Fatalf("prompt report outcome=%#v load calls=%d report=%s", outcome, loadCalls, output.String()) + } + var prompt map[string]any + if err := yaml.Unmarshal(output.Bytes(), &prompt); err != nil { + t.Fatal(err) + } + inputs := prompt["inputs"].([]any) + if len(inputs) != 2 || inputs[0].(map[string]any)["name"] != "topic" || inputs[1].(map[string]any)["name"] != "audience" || prompt["prompt_hash"] == "" { + t.Fatalf("prompt fields/order = %#v", prompt) + } + contract := prompt["output_contract"].(map[string]any) + if contract["repair_attempts"] != 0 { + t.Fatalf("zero-valued contract field was lost: %#v", contract) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 2 || !strings.Contains(output.String(), "Prompt version: \"1\"") || !strings.Contains(output.String(), `- name: "topic"`) { + t.Fatalf("default text prompt report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String()) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 3 { + t.Fatalf("profile report outcome=%#v calls=%d report=%s", outcome, loadCalls, output.String()) + } + var profile map[string]any + if err := yaml.Unmarshal(output.Bytes(), &profile); err != nil { + t.Fatal(err) + } + params := profile["effective_model_params"].(map[string]any) + if params["api_key_env"] != "PROMPTKIT_ADAPTER_TEST_KEY" || params["temperature"] != 0 || params["max_tokens"] != 0 || strings.Contains(output.String(), "credential-value-must-not-appear") { + t.Fatalf("credential or zero model fields = %#v / %s", params, output.String()) + } + extra := params["extra_params"].(map[string]any) + nested := extra["nested"].(map[string]any) + sequence := extra["sequence"].([]any) + if extra["token"] != "caller-owned-value" || nested["null_value"] != nil || nested["visible"] != true || len(sequence) != 3 || sequence[1] != nil || sequence[2] != 3 { + t.Fatalf("nested ExtraParams changed: %#v", extra) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || loadCalls != 4 || !strings.Contains(output.String(), "API key required: false") || !strings.Contains(output.String(), `api_key_env: "PROMPTKIT_ADAPTER_TEST_KEY"`) { + t.Fatalf("default text profile report = (%#v, calls=%d, %s)", outcome, loadCalls, output.String()) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "requires-key", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "api_key_required: true") { + t.Fatalf("credential requirement report = (%#v, %s)", outcome, output.String()) + } + + output.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", "empty", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(output.String(), "inputs: []") { + t.Fatalf("empty prompt input list = (%#v, %s)", outcome, output.String()) + } +} + +func TestNativeLookupErrorsAndBlankIDsReachCallbacks(t *testing.T) { + nativeErr := errors.New("native lookup failure") + calls := 0 + diagnostic := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "lookup context") + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: func(_ context.Context, request PromptRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) { + calls++ + if request.ID != "" { + t.Errorf("blank ID was rewritten to %q", request.ID) + } + return commandkit.Acquired[*promptkit.PromptInspection]{Diagnostics: []commandkit.Diagnostic{diagnostic}}, nativeErr + }})) + var output, diagnosticOutput bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "show", "prompt"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) + if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 { + t.Fatalf("missing ID was not a usage failure: (%#v, calls=%d, out=%q)", outcome, calls, output.String()) + } + output.Reset() + diagnosticOutput.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "prompt", ""}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) || calls != 1 || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "lookup context") { + t.Fatalf("native lookup failure = (%#v, calls=%d, out=%q, err=%q)", outcome, calls, output.String(), diagnosticOutput.String()) + } +} + +func TestCancelledInspectionDoesNotRender(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + called := 0 + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { + called++ + cancel() + return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil + }})) + var output bytes.Buffer + outcome := app.Execute(ctx, []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if !errors.Is(outcome.Cause, context.Canceled) || outcome.Kind != commandkit.OutcomeOperational || called != 1 || output.Len() != 0 { + t.Fatalf("cancelled inspection = (%#v, called=%d, output=%q)", outcome, called, output.String()) + } +} + +func TestDefinitionFormatPreflightAndReportDeliveryFailure(t *testing.T) { + calls := 0 + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) { + calls++ + return commandkit.Acquired[*promptkit.ProfileInspection]{Value: &promptkit.ProfileInspection{ProfileID: "default"}}, nil + }})) + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default", "--format", "json"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeUsage || calls != 0 || output.Len() != 0 { + t.Fatalf("bad format was not rejected before callback: outcome=%#v calls=%d output=%q", outcome, calls, output.String()) + } + + writeErr := errors.New("report writer failed") + outcome = app.Execute(context.Background(), []string{"promptkit", "show", "profile", "default"}, commandkit.Streams{Out: partialReportWriter{err: writeErr}, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, writeErr) || outcome.DeliveryError != writeErr || !outcome.ReportIncomplete || calls != 1 { + t.Fatalf("report delivery outcome = %#v calls=%d", outcome, calls) + } +} + +type partialReportWriter struct{ err error } + +func (writer partialReportWriter) Write(content []byte) (int, error) { + if len(content) == 0 { + return 0, writer.err + } + return 1, writer.err +} + +var _ io.Writer = partialReportWriter{} + +func TestProfileTextUsesCredentialNamesAndExplicitEmptyExtraParams(t *testing.T) { + report := profileReportFromInspection(&promptkit.ProfileInspection{ProfileID: "default", EffectiveModelParams: promptkit.ExecutionTarget{APIKeyEnv: "APP_KEY", ExtraParams: nil}}) + text, err := formatProfileText(report) + if err != nil || !strings.Contains(text, "api_key_env: \"APP_KEY\"") || !strings.Contains(text, "extra_params:\n {}\n") || !strings.Contains(text, "API key required: false") { + t.Fatalf("profile text = (%q, %v)", text, err) + } + if strings.Contains(text, "APP_KEY_VALUE") { + t.Fatalf("report contains a credential value: %q", text) + } +}