eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support internal reasoning effort overrides

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-30 02:03:04 +00:00
parent 7a00e7049c
commit f603f7ac64
10 changed files with 149 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/internal/cli.md
View File

@@ -61,6 +61,12 @@ construction errors are returned before a pipeline is prepared. Configuration
field definitions remain in [Configuration](../config.md#promptkit-profiles);
the adapter mechanics remain in [LLM Runtime](llm.md).
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
preserves inherit, replace, and clear states across the composition boundary.
Run orchestration currently passes the zero value, so production execution
inherits the selected PromptKit profile. No command flag or Notarius
configuration field exposes this internal override yet.
## Run Orchestration
After parsing and validating a run invocation, the CLI performs this ordered

7
docs/internal/llm.md
View File

@@ -36,6 +36,13 @@ validated raw bytes rather than re-encoding the decoded target. An empty
optional material is represented as one space so its named input is retained
by PromptKit.
Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer
inherits the selected profile, a non-empty value replaces it, and an empty
value clears inherited reasoning. Ordinary CLI execution currently supplies no
override, so profile behavior remains unchanged.
An empty request profile lets the prompt select its configured default. The CLI
prepares every explicitly selected binding profile before a run begins, so a
missing explicit profile fails before stage execution. Calls record the profile

19
internal/cli/catalog.go
View File

@@ -143,7 +143,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
registries.Outputs == nil
}
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
@@ -151,16 +151,16 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
if err != nil {
return nil, nil, err
}
return buildProductionLLMClient(ctx, cfg, profileID, assets)
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
}
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, assets)
return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
}
}
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
@@ -169,10 +169,11 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
Assets: assets,
Recorder: recorder,
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
Assets: assets,
Recorder: recorder,
ReasoningEffort: overrides.ReasoningEffort,
})
if err != nil {
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)

2
internal/cli/npc_registry_contract_test.go
View File

@@ -51,7 +51,7 @@ pipelines:
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},

12
internal/cli/production_contract_test.go
View File

@@ -322,7 +322,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
@@ -381,7 +381,7 @@ func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
}
for _, tt := range factories {
t.Run(tt.name, func(t *testing.T) {
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile")
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
@@ -410,14 +410,14 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile")
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
}
})
t.Run("nil assets", func(t *testing.T) {
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
@@ -427,7 +427,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Concurrency.TotalLLM = 0
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
}
@@ -682,7 +682,7 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return fake, nil, nil
}
return options

8
internal/cli/run.go
View File

@@ -48,7 +48,11 @@ type Options struct {
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
type LLMRuntimeOverrides struct {
ReasoningEffort *string
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
// Run executes the command-line interface and returns a process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
@@ -364,7 +368,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{})
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}

13
internal/cli/run_contract_test.go
View File

@@ -245,8 +245,10 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
harness := newStateTestHarness()
var factoryProfiles []string
opts := harness.options()
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
var factoryOverrides []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
factoryOverrides = append(factoryOverrides, overrides)
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
@@ -257,6 +259,9 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
}
harness.mu.Lock()
profiles := append([]string(nil), harness.moduleProfiles...)
harness.mu.Unlock()
@@ -279,7 +284,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
opts := harness.options()
registerRunContractValidator(t, &opts, &validatorProfiles)
factoryProfiles := []string{}
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
return nil, nil, nil
}
@@ -302,7 +307,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
@@ -376,7 +381,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
t.Run("LLM factory", func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, errors.New("injected LLM factory failure")
}
var stdout, stderr bytes.Buffer

2
internal/cli/state_hardening_test.go
View File

@@ -848,7 +848,7 @@ func (h *stateTestHarness) options() Options {
defer h.mu.Unlock()
h.runIDCalls++
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, nil
}}
}

46
internal/framework/llm/promptkit_client.go
View File

@@ -19,20 +19,22 @@ import (
const promptKitProviderName = "promptkit"
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ReasoningEffort *string
}
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
reasoningEffort *string
}
type LLMProfileRecorder struct {
@@ -71,11 +73,17 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
var reasoningEffort *string
if cfg.ReasoningEffort != nil {
value := *cfg.ReasoningEffort
reasoningEffort = &value
}
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
reasoningEffort: reasoningEffort,
}, nil
}
@@ -94,6 +102,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
sessionID := strings.TrimSpace(req.SessionID)
var execution *promptkit.ExecutionTargetOverride
if c.reasoningEffort != nil {
reasoningEffort := *c.reasoningEffort
execution = &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoningEffort,
}
}
runReq := promptkit.RunRequest{
PromptID: promptID,
@@ -102,6 +117,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {

79
internal/framework/llm/promptkit_client_test.go
View File

@@ -150,6 +150,64 @@ func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) {
}
}
func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
tests := []struct {
name string
override func() *string
mutateAfterCreate bool
want string
}{
{
name: "inherit",
want: "profile-reasoning",
},
{
name: "replace",
override: func() *string { value := "focused"; return &value },
want: "focused",
},
{
name: "clear",
override: func() *string { value := ""; return &value },
want: "",
},
{
name: "defensive copy",
override: func() *string { value := "original"; return &value },
mutateAfterCreate: true,
want: "original",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
var override *string
if tt.override != nil {
override = tt.override()
}
client := newTestPromptKitClientWithReasoning(t, fake, override)
if tt.mutateAfterCreate {
*override = "mutated"
}
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.direct-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
Vars: map[string]any{"custom": "value"},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want {
t.Fatalf("reasoning effort = %q, want %q", got, tt.want)
}
})
}
}
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
registry := NewAssetRegistry()
@@ -464,21 +522,28 @@ func TestPromptKitClientValidatesRequest(t *testing.T) {
}
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
return newTestPromptKitClientWithReasoning(t, fake, nil)
}
func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient {
t.Helper()
registry := newTestPromptKitAssets(t)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
Assets: registry,
ReasoningEffort: reasoningEffort,
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
ReasoningEffort: "profile-reasoning",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
ReasoningEffort: "profile-reasoning",
}),
),
promptkit.WithLLMClient(fake),