Support internal reasoning effort overrides
This commit is contained in:
@@ -61,6 +61,12 @@ construction errors are returned before a pipeline is prepared. Configuration
|
|||||||
field definitions remain in [Configuration](../config.md#promptkit-profiles);
|
field definitions remain in [Configuration](../config.md#promptkit-profiles);
|
||||||
the adapter mechanics remain in [LLM Runtime](llm.md).
|
the adapter mechanics remain in [LLM Runtime](llm.md).
|
||||||
|
|
||||||
|
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
|
||||||
|
preserves inherit, replace, and clear states across the composition boundary.
|
||||||
|
Run orchestration currently passes the zero value, so production execution
|
||||||
|
inherits the selected PromptKit profile. No command flag or Notarius
|
||||||
|
configuration field exposes this internal override yet.
|
||||||
|
|
||||||
## Run Orchestration
|
## Run Orchestration
|
||||||
|
|
||||||
After parsing and validating a run invocation, the CLI performs this ordered
|
After parsing and validating a run invocation, the CLI performs this ordered
|
||||||
|
|||||||
@@ -36,6 +36,13 @@ validated raw bytes rather than re-encoding the decoded target. An empty
|
|||||||
optional material is represented as one space so its named input is retained
|
optional material is represented as one space so its named input is retained
|
||||||
by PromptKit.
|
by PromptKit.
|
||||||
|
|
||||||
|
Client construction may also receive a run-wide reasoning-effort override from
|
||||||
|
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
||||||
|
creates a fresh PromptKit execution override for each request: a nil pointer
|
||||||
|
inherits the selected profile, a non-empty value replaces it, and an empty
|
||||||
|
value clears inherited reasoning. Ordinary CLI execution currently supplies no
|
||||||
|
override, so profile behavior remains unchanged.
|
||||||
|
|
||||||
An empty request profile lets the prompt select its configured default. The CLI
|
An empty request profile lets the prompt select its configured default. The CLI
|
||||||
prepares every explicitly selected binding profile before a run begins, so a
|
prepares every explicitly selected binding profile before a run begins, so a
|
||||||
missing explicit profile fails before stage execution. Calls record the profile
|
missing explicit profile fails before stage execution. Calls record the profile
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
|||||||
registries.Outputs == nil
|
registries.Outputs == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -151,16 +151,16 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return buildProductionLLMClient(ctx, cfg, profileID, assets)
|
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
|
||||||
}
|
}
|
||||||
|
|
||||||
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
|
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
|
||||||
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
return buildProductionLLMClient(ctx, cfg, profileID, assets)
|
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -169,10 +169,11 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
|
|||||||
}
|
}
|
||||||
recorder := llm.NewLLMProfileRecorder()
|
recorder := llm.NewLLMProfileRecorder()
|
||||||
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
|
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
|
||||||
ProfileDir: cfg.PromptKit.ProfileDir,
|
ProfileDir: cfg.PromptKit.ProfileDir,
|
||||||
ProfileFile: cfg.PromptKit.ProfileFile,
|
ProfileFile: cfg.PromptKit.ProfileFile,
|
||||||
Assets: assets,
|
Assets: assets,
|
||||||
Recorder: recorder,
|
Recorder: recorder,
|
||||||
|
ReasoningEffort: overrides.ReasoningEffort,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
|
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ pipelines:
|
|||||||
options := Options{
|
options := Options{
|
||||||
Catalog: catalogFromRegistries(components.registries),
|
Catalog: catalogFromRegistries(components.registries),
|
||||||
Registries: components.registries,
|
Registries: components.registries,
|
||||||
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
llmConstructed = true
|
llmConstructed = true
|
||||||
return nil, nil, errors.New("LLM client must not be constructed")
|
return nil, nil, errors.New("LLM client must not be constructed")
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -322,7 +322,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
|
|||||||
options := Options{
|
options := Options{
|
||||||
Catalog: catalogFromRegistries(components.registries),
|
Catalog: catalogFromRegistries(components.registries),
|
||||||
Registries: components.registries,
|
Registries: components.registries,
|
||||||
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
llmConstructed = true
|
llmConstructed = true
|
||||||
return nil, nil, errors.New("LLM client must not be constructed")
|
return nil, nil, errors.New("LLM client must not be constructed")
|
||||||
},
|
},
|
||||||
@@ -381,7 +381,7 @@ func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, tt := range factories {
|
for _, tt := range factories {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile")
|
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("build production LLM runtime: %v", err)
|
t.Fatalf("build production LLM runtime: %v", err)
|
||||||
}
|
}
|
||||||
@@ -410,14 +410,14 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
|
|||||||
t.Run("canceled context", func(t *testing.T) {
|
t.Run("canceled context", func(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile")
|
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
|
||||||
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
|
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
|
||||||
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
|
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("nil assets", func(t *testing.T) {
|
t.Run("nil assets", func(t *testing.T) {
|
||||||
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile")
|
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
|
||||||
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
|
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
|
||||||
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
|
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
|
||||||
}
|
}
|
||||||
@@ -427,7 +427,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
|
|||||||
components := productionTestComponents(t)
|
components := productionTestComponents(t)
|
||||||
cfg := config.Default()
|
cfg := config.Default()
|
||||||
cfg.Concurrency.TotalLLM = 0
|
cfg.Concurrency.TotalLLM = 0
|
||||||
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile")
|
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
|
||||||
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
|
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
|
||||||
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
|
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
|
||||||
}
|
}
|
||||||
@@ -682,7 +682,7 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
|
|||||||
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
|
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
|
||||||
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
|
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
|
||||||
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
|
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
|
||||||
options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
return fake, nil, nil
|
return fake, nil, nil
|
||||||
}
|
}
|
||||||
return options
|
return options
|
||||||
|
|||||||
@@ -48,7 +48,11 @@ type Options struct {
|
|||||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
|
type LLMRuntimeOverrides struct {
|
||||||
|
ReasoningEffort *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
|
||||||
|
|
||||||
// Run executes the command-line interface and returns a process exit code.
|
// Run executes the command-line interface and returns a process exit code.
|
||||||
func Run(args []string, stdout, stderr io.Writer) int {
|
func Run(args []string, stdout, stderr io.Writer) int {
|
||||||
@@ -364,7 +368,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if len(profileIDs) == 1 {
|
if len(profileIDs) == 1 {
|
||||||
factoryProfileID = profileIDs[0]
|
factoryProfileID = profileIDs[0]
|
||||||
}
|
}
|
||||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
|
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -245,8 +245,10 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
|||||||
harness := newStateTestHarness()
|
harness := newStateTestHarness()
|
||||||
var factoryProfiles []string
|
var factoryProfiles []string
|
||||||
opts := harness.options()
|
opts := harness.options()
|
||||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
var factoryOverrides []LLMRuntimeOverrides
|
||||||
|
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
factoryProfiles = append(factoryProfiles, profileID)
|
factoryProfiles = append(factoryProfiles, profileID)
|
||||||
|
factoryOverrides = append(factoryOverrides, overrides)
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -257,6 +259,9 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
|||||||
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
||||||
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
||||||
}
|
}
|
||||||
|
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
|
||||||
|
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
|
||||||
|
}
|
||||||
harness.mu.Lock()
|
harness.mu.Lock()
|
||||||
profiles := append([]string(nil), harness.moduleProfiles...)
|
profiles := append([]string(nil), harness.moduleProfiles...)
|
||||||
harness.mu.Unlock()
|
harness.mu.Unlock()
|
||||||
@@ -279,7 +284,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
|||||||
opts := harness.options()
|
opts := harness.options()
|
||||||
registerRunContractValidator(t, &opts, &validatorProfiles)
|
registerRunContractValidator(t, &opts, &validatorProfiles)
|
||||||
factoryProfiles := []string{}
|
factoryProfiles := []string{}
|
||||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
factoryProfiles = append(factoryProfiles, profileID)
|
factoryProfiles = append(factoryProfiles, profileID)
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
@@ -302,7 +307,7 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
|||||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||||
factoryCalls := 0
|
factoryCalls := 0
|
||||||
opts := newStateTestHarness().options()
|
opts := newStateTestHarness().options()
|
||||||
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
factoryCalls++
|
factoryCalls++
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
@@ -376,7 +381,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
|
|||||||
t.Run("LLM factory", func(t *testing.T) {
|
t.Run("LLM factory", func(t *testing.T) {
|
||||||
roots := newStateTestRoots(t)
|
roots := newStateTestRoots(t)
|
||||||
opts := newStateTestHarness().options()
|
opts := newStateTestHarness().options()
|
||||||
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
return nil, nil, errors.New("injected LLM factory failure")
|
return nil, nil, errors.New("injected LLM factory failure")
|
||||||
}
|
}
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|||||||
@@ -848,7 +848,7 @@ func (h *stateTestHarness) options() Options {
|
|||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
h.runIDCalls++
|
h.runIDCalls++
|
||||||
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
|
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
|
||||||
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,20 +19,22 @@ import (
|
|||||||
const promptKitProviderName = "promptkit"
|
const promptKitProviderName = "promptkit"
|
||||||
|
|
||||||
type PromptKitClientConfig struct {
|
type PromptKitClientConfig struct {
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
ProfileFile string
|
ProfileFile string
|
||||||
Assets *AssetRegistry
|
Assets *AssetRegistry
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
EngineOptions []promptkit.Option
|
EngineOptions []promptkit.Option
|
||||||
Recorder *LLMProfileRecorder
|
Recorder *LLMProfileRecorder
|
||||||
|
ReasoningEffort *string
|
||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitClient struct {
|
type PromptKitClient struct {
|
||||||
engine *promptkit.Engine
|
engine *promptkit.Engine
|
||||||
recorder *LLMProfileRecorder
|
recorder *LLMProfileRecorder
|
||||||
profileDir string
|
profileDir string
|
||||||
profileFile string
|
profileFile string
|
||||||
|
reasoningEffort *string
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMProfileRecorder struct {
|
type LLMProfileRecorder struct {
|
||||||
@@ -71,11 +73,17 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
if recorder == nil {
|
if recorder == nil {
|
||||||
recorder = NewLLMProfileRecorder()
|
recorder = NewLLMProfileRecorder()
|
||||||
}
|
}
|
||||||
|
var reasoningEffort *string
|
||||||
|
if cfg.ReasoningEffort != nil {
|
||||||
|
value := *cfg.ReasoningEffort
|
||||||
|
reasoningEffort = &value
|
||||||
|
}
|
||||||
return &PromptKitClient{
|
return &PromptKitClient{
|
||||||
engine: engine,
|
engine: engine,
|
||||||
recorder: recorder,
|
recorder: recorder,
|
||||||
profileDir: strings.TrimSpace(cfg.ProfileDir),
|
profileDir: strings.TrimSpace(cfg.ProfileDir),
|
||||||
profileFile: strings.TrimSpace(cfg.ProfileFile),
|
profileFile: strings.TrimSpace(cfg.ProfileFile),
|
||||||
|
reasoningEffort: reasoningEffort,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,6 +102,13 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||||
}
|
}
|
||||||
sessionID := strings.TrimSpace(req.SessionID)
|
sessionID := strings.TrimSpace(req.SessionID)
|
||||||
|
var execution *promptkit.ExecutionTargetOverride
|
||||||
|
if c.reasoningEffort != nil {
|
||||||
|
reasoningEffort := *c.reasoningEffort
|
||||||
|
execution = &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: &reasoningEffort,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
runReq := promptkit.RunRequest{
|
runReq := promptkit.RunRequest{
|
||||||
PromptID: promptID,
|
PromptID: promptID,
|
||||||
@@ -102,6 +117,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
|
|||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
Inputs: promptKitInputs(req.Inputs),
|
Inputs: promptKitInputs(req.Inputs),
|
||||||
Vars: promptKitVars(req, sessionID),
|
Vars: promptKitVars(req, sessionID),
|
||||||
|
Execution: execution,
|
||||||
}
|
}
|
||||||
prepared, err := c.engine.Prepare(ctx, runReq)
|
prepared, err := c.engine.Prepare(ctx, runReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -150,6 +150,64 @@ func TestPromptKitClientDoesNotInventDirectSession(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override func() *string
|
||||||
|
mutateAfterCreate bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "inherit",
|
||||||
|
want: "profile-reasoning",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "replace",
|
||||||
|
override: func() *string { value := "focused"; return &value },
|
||||||
|
want: "focused",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "clear",
|
||||||
|
override: func() *string { value := ""; return &value },
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "defensive copy",
|
||||||
|
override: func() *string { value := "original"; return &value },
|
||||||
|
mutateAfterCreate: true,
|
||||||
|
want: "original",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
var override *string
|
||||||
|
if tt.override != nil {
|
||||||
|
override = tt.override()
|
||||||
|
}
|
||||||
|
client := newTestPromptKitClientWithReasoning(t, fake, override)
|
||||||
|
if tt.mutateAfterCreate {
|
||||||
|
*override = "mutated"
|
||||||
|
}
|
||||||
|
|
||||||
|
var out map[string]any
|
||||||
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "adapter.direct-session",
|
||||||
|
Inputs: contracts.LLMInputSet{
|
||||||
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
|
},
|
||||||
|
Vars: map[string]any{"custom": "value"},
|
||||||
|
}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := fake.lastRequest().Target.ReasoningEffort; got != tt.want {
|
||||||
|
t.Fatalf("reasoning effort = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
||||||
t.Run("assets", func(t *testing.T) {
|
t.Run("assets", func(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
@@ -464,21 +522,28 @@ func TestPromptKitClientValidatesRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
|
func newTestPromptKitClient(t *testing.T, fake *fakePromptKitLLM) *PromptKitClient {
|
||||||
|
return newTestPromptKitClientWithReasoning(t, fake, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, reasoningEffort *string) *PromptKitClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
registry := newTestPromptKitAssets(t)
|
registry := newTestPromptKitAssets(t)
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: registry,
|
Assets: registry,
|
||||||
|
ReasoningEffort: reasoningEffort,
|
||||||
EngineOptions: []promptkit.Option{
|
EngineOptions: []promptkit.Option{
|
||||||
promptkit.WithProfiles(
|
promptkit.WithProfiles(
|
||||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
ID: "default-profile",
|
ID: "default-profile",
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
Endpoint: "http://127.0.0.1:1/v1",
|
||||||
Model: "default-model",
|
Model: "default-model",
|
||||||
|
ReasoningEffort: "profile-reasoning",
|
||||||
}),
|
}),
|
||||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
ID: "explicit-profile",
|
ID: "explicit-profile",
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
Endpoint: "http://127.0.0.1:1/v1",
|
||||||
Model: "explicit-model",
|
Model: "explicit-model",
|
||||||
|
ReasoningEffort: "profile-reasoning",
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
promptkit.WithLLMClient(fake),
|
promptkit.WithLLMClient(fake),
|
||||||
|
|||||||
Reference in New Issue
Block a user