eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support internal reasoning effort overrides

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-30 02:03:04 +00:00
parent 7a00e7049c
commit f603f7ac64
10 changed files with 149 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
internal/cli/catalog.go
View File

@@ -143,7 +143,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
registries.Outputs == nil
}
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
@@ -151,16 +151,16 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
if err != nil {
return nil, nil, err
}
return buildProductionLLMClient(ctx, cfg, profileID, assets)
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
}
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, assets)
return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
}
}
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
@@ -169,10 +169,11 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
Assets: assets,
Recorder: recorder,
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
Assets: assets,
Recorder: recorder,
ReasoningEffort: overrides.ReasoningEffort,
})
if err != nil {
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)

2
internal/cli/npc_registry_contract_test.go
View File

@@ -51,7 +51,7 @@ pipelines:
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},

12
internal/cli/production_contract_test.go
View File

@@ -322,7 +322,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
@@ -381,7 +381,7 @@ func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
}
for _, tt := range factories {
t.Run(tt.name, func(t *testing.T) {
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile")
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
@@ -410,14 +410,14 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile")
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
}
})
t.Run("nil assets", func(t *testing.T) {
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
@@ -427,7 +427,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Concurrency.TotalLLM = 0
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
}
@@ -682,7 +682,7 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return fake, nil, nil
}
return options

8
internal/cli/run.go
View File

@@ -48,7 +48,11 @@ type Options struct {
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
type LLMRuntimeOverrides struct {
ReasoningEffort *string
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
// Run executes the command-line interface and returns a process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
@@ -364,7 +368,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{})
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}

13
internal/cli/run_contract_test.go
View File

@@ -245,8 +245,10 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
harness := newStateTestHarness()
var factoryProfiles []string
opts := harness.options()
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
var factoryOverrides []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
factoryOverrides = append(factoryOverrides, overrides)
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
@@ -257,6 +259,9 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
}
harness.mu.Lock()
profiles := append([]string(nil), harness.moduleProfiles...)
harness.mu.Unlock()
@@ -279,7 +284,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
opts := harness.options()
registerRunContractValidator(t, &opts, &validatorProfiles)
factoryProfiles := []string{}
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
return nil, nil, nil
}
@@ -302,7 +307,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
@@ -376,7 +381,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
t.Run("LLM factory", func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, errors.New("injected LLM factory failure")
}
var stdout, stderr bytes.Buffer

2
internal/cli/state_hardening_test.go
View File

@@ -848,7 +848,7 @@ func (h *stateTestHarness) options() Options {
defer h.mu.Unlock()
h.runIDCalls++
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, nil
}}
}

46
internal/framework/llm/promptkit_client.go
View File

@@ -19,20 +19,22 @@ import (
const promptKitProviderName = "promptkit"
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ReasoningEffort *string
}
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
reasoningEffort *string
}
type LLMProfileRecorder struct {
@@ -71,11 +73,17 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
var reasoningEffort *string
if cfg.ReasoningEffort != nil {
value := *cfg.ReasoningEffort
reasoningEffort = &value
}
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
reasoningEffort: reasoningEffort,
}, nil
}
@@ -94,6 +102,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
sessionID := strings.TrimSpace(req.SessionID)
var execution *promptkit.ExecutionTargetOverride
if c.reasoningEffort != nil {
reasoningEffort := *c.reasoningEffort
execution = &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoningEffort,
}
}
runReq := promptkit.RunRequest{
PromptID: promptID,
@@ -102,6 +117,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {

79
internal/framework/llm/promptkit_client_test.go
View File

@@ -150,6 +150,64 @@ func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) {
}
}
func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
tests := []struct {
name string
override func() *string
mutateAfterCreate bool
want string
}{
{
name: "inherit",
want: "profile-reasoning",
},
{
name: "replace",
override: func() *string { value := "focused"; return &value },
want: "focused",
},
{
name: "clear",
override: func() *string { value := ""; return &value },
want: "",
},
{
name: "defensive copy",
override: func() *string { value := "original"; return &value },
mutateAfterCreate: true,
want: "original",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fake := &fakePromptKitLLM{content: `{"ok":true}`}
var override *string
if tt.override != nil {
override = tt.override()
}
client := newTestPromptKitClientWithReasoning(t, fake, override)
if tt.mutateAfterCreate {
*override = "mutated"
}
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.direct-session",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
Vars: map[string]any{"custom": "value"},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want {
t.Fatalf("reasoning effort = %q, want %q", got, tt.want)
}
})
}
}
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
t.Run("assets", func(t *testing.T) {
registry := NewAssetRegistry()
@@ -464,21 +522,28 @@ func TestPromptKitClientValidatesRequest(t *testing.T) {
}
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
return newTestPromptKitClientWithReasoning(t, fake, nil)
}
func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient {
t.Helper()
registry := newTestPromptKitAssets(t)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: registry,
Assets: registry,
ReasoningEffort: reasoningEffort,
EngineOptions: []promptkit.Option{
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
ReasoningEffort: "profile-reasoning",
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
ReasoningEffort: "profile-reasoning",
}),
),
promptkit.WithLLMClient(fake),