diff --git a/docs/internal/cli.md b/docs/internal/cli.md index 5a4a718..b45059b 100644 --- a/docs/internal/cli.md +++ b/docs/internal/cli.md @@ -61,6 +61,12 @@ construction errors are returned before a pipeline is prepared. Configuration field definitions remain in [Configuration](../config.md#promptkit-profiles); the adapter mechanics remain in [LLM Runtime](llm.md). +The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer +preserves inherit, replace, and clear states across the composition boundary. +Run orchestration currently passes the zero value, so production execution +inherits the selected PromptKit profile. No command flag or Notarius +configuration field exposes this internal override yet. + ## Run Orchestration After parsing and validating a run invocation, the CLI performs this ordered diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 2e9c915..53e4d00 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -36,6 +36,13 @@ validated raw bytes rather than re-encoding the decoded target. An empty optional material is represented as one space so its named input is retained by PromptKit. +Client construction may also receive a run-wide reasoning-effort override from +the CLI factory boundary. The adapter copies the caller-owned pointer and +creates a fresh PromptKit execution override for each request: a nil pointer +inherits the selected profile, a non-empty value replaces it, and an empty +value clears inherited reasoning. Ordinary CLI execution currently supplies no +override, so profile behavior remains unchanged. + An empty request profile lets the prompt select its configured default. The CLI prepares every explicitly selected binding profile before a run begins, so a missing explicit profile fails before stage execution. Calls record the profile diff --git a/internal/cli/catalog.go b/internal/cli/catalog.go index 3e90ea8..3ecdcbe 100644 --- a/internal/cli/catalog.go +++ b/internal/cli/catalog.go @@ -143,7 +143,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool { registries.Outputs == nil } -func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { +func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { if err := ctx.Err(); err != nil { return nil, nil, err } @@ -151,16 +151,16 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI if err != nil { return nil, nil, err } - return buildProductionLLMClient(ctx, cfg, profileID, assets) + return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets) } func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory { - return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - return buildProductionLLMClient(ctx, cfg, profileID, assets) + return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets) } } -func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { +func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { if err := ctx.Err(); err != nil { return nil, nil, err } @@ -169,10 +169,11 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID } recorder := llm.NewLLMProfileRecorder() client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{ - ProfileDir: cfg.PromptKit.ProfileDir, - ProfileFile: cfg.PromptKit.ProfileFile, - Assets: assets, - Recorder: recorder, + ProfileDir: cfg.PromptKit.ProfileDir, + ProfileFile: cfg.PromptKit.ProfileFile, + Assets: assets, + Recorder: recorder, + ReasoningEffort: overrides.ReasoningEffort, }) if err != nil { return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err) diff --git a/internal/cli/npc_registry_contract_test.go b/internal/cli/npc_registry_contract_test.go index a891e01..156d4b7 100644 --- a/internal/cli/npc_registry_contract_test.go +++ b/internal/cli/npc_registry_contract_test.go @@ -51,7 +51,7 @@ pipelines: options := Options{ Catalog: catalogFromRegistries(components.registries), Registries: components.registries, - LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { llmConstructed = true return nil, nil, errors.New("LLM client must not be constructed") }, diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index 7696479..efe7324 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -322,7 +322,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution options := Options{ Catalog: catalogFromRegistries(components.registries), Registries: components.registries, - LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { llmConstructed = true return nil, nil, errors.New("LLM client must not be constructed") }, @@ -381,7 +381,7 @@ func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) { } for _, tt := range factories { t.Run(tt.name, func(t *testing.T) { - client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile") + client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) if err != nil { t.Fatalf("build production LLM runtime: %v", err) } @@ -410,14 +410,14 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) { t.Run("canceled context", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile") + client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{}) if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err) } }) t.Run("nil assets", func(t *testing.T) { - client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile") + client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err) } @@ -427,7 +427,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Concurrency.TotalLLM = 0 - client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile") + client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{}) if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err) } @@ -682,7 +682,7 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options { options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() } options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil } options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") } - options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return fake, nil, nil } return options diff --git a/internal/cli/run.go b/internal/cli/run.go index fee1fa3..22a3794 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -48,7 +48,11 @@ type Options struct { DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter } -type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) +type LLMRuntimeOverrides struct { + ReasoningEffort *string +} + +type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) // Run executes the command-line interface and returns a process exit code. func Run(args []string, stdout, stderr io.Writer) int { @@ -364,7 +368,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if len(profileIDs) == 1 { factoryProfileID = profileIDs[0] } - llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID) + llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{}) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } diff --git a/internal/cli/run_contract_test.go b/internal/cli/run_contract_test.go index f2cb489..b27bc07 100644 --- a/internal/cli/run_contract_test.go +++ b/internal/cli/run_contract_test.go @@ -245,8 +245,10 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { harness := newStateTestHarness() var factoryProfiles []string opts := harness.options() - opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + var factoryOverrides []LLMRuntimeOverrides + opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { factoryProfiles = append(factoryProfiles, profileID) + factoryOverrides = append(factoryOverrides, overrides) return nil, nil, nil } var stdout, stderr bytes.Buffer @@ -257,6 +259,9 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" { t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles) } + if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil { + t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides) + } harness.mu.Lock() profiles := append([]string(nil), harness.moduleProfiles...) harness.mu.Unlock() @@ -279,7 +284,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { opts := harness.options() registerRunContractValidator(t, &opts, &validatorProfiles) factoryProfiles := []string{} - opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { factoryProfiles = append(factoryProfiles, profileID) return nil, nil, nil } @@ -302,7 +307,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir)) factoryCalls := 0 opts := newStateTestHarness().options() - opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { factoryCalls++ return nil, nil, nil } @@ -376,7 +381,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) { t.Run("LLM factory", func(t *testing.T) { roots := newStateTestRoots(t) opts := newStateTestHarness().options() - opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return nil, nil, errors.New("injected LLM factory failure") } var stdout, stderr bytes.Buffer diff --git a/internal/cli/state_hardening_test.go b/internal/cli/state_hardening_test.go index bf9e301..029876c 100644 --- a/internal/cli/state_hardening_test.go +++ b/internal/cli/state_hardening_test.go @@ -848,7 +848,7 @@ func (h *stateTestHarness) options() Options { defer h.mu.Unlock() h.runIDCalls++ return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil - }, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + }, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return nil, nil, nil }} } diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index d1ccfa5..d036080 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -19,20 +19,22 @@ import ( const promptKitProviderName = "promptkit" type PromptKitClientConfig struct { - ProfileDir string - ProfileFile string - Assets *AssetRegistry - Timeout time.Duration - HTTPClient *http.Client - EngineOptions []promptkit.Option - Recorder *LLMProfileRecorder + ProfileDir string + ProfileFile string + Assets *AssetRegistry + Timeout time.Duration + HTTPClient *http.Client + EngineOptions []promptkit.Option + Recorder *LLMProfileRecorder + ReasoningEffort *string } type PromptKitClient struct { - engine *promptkit.Engine - recorder *LLMProfileRecorder - profileDir string - profileFile string + engine *promptkit.Engine + recorder *LLMProfileRecorder + profileDir string + profileFile string + reasoningEffort *string } type LLMProfileRecorder struct { @@ -71,11 +73,17 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) { if recorder == nil { recorder = NewLLMProfileRecorder() } + var reasoningEffort *string + if cfg.ReasoningEffort != nil { + value := *cfg.ReasoningEffort + reasoningEffort = &value + } return &PromptKitClient{ - engine: engine, - recorder: recorder, - profileDir: strings.TrimSpace(cfg.ProfileDir), - profileFile: strings.TrimSpace(cfg.ProfileFile), + engine: engine, + recorder: recorder, + profileDir: strings.TrimSpace(cfg.ProfileDir), + profileFile: strings.TrimSpace(cfg.ProfileFile), + reasoningEffort: reasoningEffort, }, nil } @@ -94,6 +102,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty") } sessionID := strings.TrimSpace(req.SessionID) + var execution *promptkit.ExecutionTargetOverride + if c.reasoningEffort != nil { + reasoningEffort := *c.reasoningEffort + execution = &promptkit.ExecutionTargetOverride{ + ReasoningEffort: &reasoningEffort, + } + } runReq := promptkit.RunRequest{ PromptID: promptID, @@ -102,6 +117,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. SessionID: sessionID, Inputs: promptKitInputs(req.Inputs), Vars: promptKitVars(req, sessionID), + Execution: execution, } prepared, err := c.engine.Prepare(ctx, runReq) if err != nil { diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 8ab6b6e..c1ac236 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -150,6 +150,64 @@ func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) { } } +func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) { + tests := []struct { + name string + override func() *string + mutateAfterCreate bool + want string + }{ + { + name: "inherit", + want: "profile-reasoning", + }, + { + name: "replace", + override: func() *string { value := "focused"; return &value }, + want: "focused", + }, + { + name: "clear", + override: func() *string { value := ""; return &value }, + want: "", + }, + { + name: "defensive copy", + override: func() *string { value := "original"; return &value }, + mutateAfterCreate: true, + want: "original", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := &fakePromptKitLLM{content: `{"ok":true}`} + var override *string + if tt.override != nil { + override = tt.override() + } + client := newTestPromptKitClientWithReasoning(t, fake, override) + if tt.mutateAfterCreate { + *override = "mutated" + } + + var out map[string]any + _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "adapter.direct-session", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + Vars: map[string]any{"custom": "value"}, + }, &out) + if err != nil { + t.Fatalf("CompleteStructured() error = %v, want nil", err) + } + if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want { + t.Fatalf("reasoning effort = %q, want %q", got, tt.want) + } + }) + } +} + func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) { t.Run("assets", func(t *testing.T) { registry := NewAssetRegistry() @@ -464,21 +522,28 @@ func TestPromptKitClientValidatesRequest(t *testing.T) { } func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient { + return newTestPromptKitClientWithReasoning(t, fake, nil) +} + +func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient { t.Helper() registry := newTestPromptKitAssets(t) client, err := NewPromptKitClient(PromptKitClientConfig{ - Assets: registry, + Assets: registry, + ReasoningEffort: reasoningEffort, EngineOptions: []promptkit.Option{ promptkit.WithProfiles( promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "default-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "default-model", + ID: "default-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "default-model", + ReasoningEffort: "profile-reasoning", }), promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "explicit-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "explicit-model", + ID: "explicit-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "explicit-model", + ReasoningEffort: "profile-reasoning", }), ), promptkit.WithLLMClient(fake),