eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expose run-wide reasoning effort controls

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-30 02:11:35 +00:00
parent f603f7ac64
commit f8333f2c15
9 changed files with 269 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
internal/cli/run.go
View File

@@ -142,13 +142,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debug := fs.Bool("debug", false, "write a debug bundle")
debugDir := fs.String("debug-dir", "", "debug bundle directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
recomputeStep := singleValueFlag{}
recomputeStep := singleValueFlag{name: "--recompute-step"}
chunkCache := chunkCacheFlag{}
sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&sessionID, "session-id", "prompt session identifier")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
@@ -194,6 +197,22 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
return 2
}
if reasoningEffort.set && *clearReasoningEffort {
fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
return 2
}
if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" {
fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty")
return 2
}
runtimeOverrides := LLMRuntimeOverrides{}
if reasoningEffort.set {
value := strings.TrimSpace(reasoningEffort.value)
runtimeOverrides.ReasoningEffort = &value
} else if *clearReasoningEffort {
value := ""
runtimeOverrides.ReasoningEffort = &value
}
only, err := parseOnly(*onlyRaw)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -284,17 +303,18 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
}
invocation := debugbundle.Invocation{
Operation: "run",
PipelineID: pipelineID,
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
Operation: "run",
PipelineID: pipelineID,
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
}
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
@@ -368,7 +388,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{})
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
@@ -392,7 +412,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), runtimeOverrides, *resume)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -499,6 +519,7 @@ func checkpointHandlersForRun(
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
runtimeOverrides LLMRuntimeOverrides,
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
if !settings.Enabled {
@@ -512,7 +533,7 @@ func checkpointHandlersForRun(
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: combineCheckpointFingerprints(
llmProfileFingerprints(llmProfiles),
@@ -673,7 +694,7 @@ func rawInputDigest(data []byte) string {
return "sha256:" + hex.EncodeToString(sum[:])
}
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []checkpoint.Fingerprint {
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
var values []checkpoint.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
@@ -681,6 +702,13 @@ func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []
if strings.TrimSpace(sessionID) != "" {
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
}
if runtimeOverrides.ReasoningEffort != nil {
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
if value == "" {
value = "<cleared>"
}
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
}
return values
}
@@ -837,7 +865,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
return true
default:
return false
@@ -897,11 +925,11 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--session-id" {
if arg != "--session-id" && arg != "--reasoning-effort" {
continue
}
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
return fmt.Errorf("flag needs an argument: --session-id")
return fmt.Errorf("flag needs an argument: %s", arg)
}
}
return nil
@@ -1185,6 +1213,7 @@ type sessionIDFlag struct {
}
type singleValueFlag struct {
name string
value string
set bool
}
@@ -1198,7 +1227,7 @@ func (flag *singleValueFlag) String() string {
func (flag *singleValueFlag) Set(value string) error {
if flag.set {
return fmt.Errorf("--recompute-step may be specified only once")
return fmt.Errorf("%s may be specified only once", flag.name)
}
flag.value = value
flag.set = true

124
internal/cli/run_contract_test.go
View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -319,6 +320,129 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
})
}
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
tests := []struct {
name string
flags []string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
var got []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
got = append(got, overrides)
return nil, nil, nil
}
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(got) != 1 {
t.Fatalf("factory overrides = %#v, want one call", got)
}
if !tt.wantSet {
if got[0].ReasoningEffort != nil {
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
}
return
}
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
}
})
}
}
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
tests := []struct {
name string
flags []string
wantError string
}{
{
name: "mutually exclusive controls",
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
wantError: "cannot be combined",
},
{
name: "empty replacement",
flags: []string{"--reasoning-effort", " "},
wantError: "must not be empty",
},
{
name: "duplicate replacement",
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
wantError: "may be specified only once",
},
{
name: "missing replacement",
flags: []string{"--reasoning-effort"},
wantError: "flag needs an argument",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
}
}
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
replacement := " focused "
cleared := ""
states := []struct {
name string
overrides LLMRuntimeOverrides
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
}
digests := make(map[string]string, len(states))
for _, state := range states {
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
var value string
var found bool
for _, fingerprint := range fingerprints {
if fingerprint.Name == "reasoning_effort_override" {
value, found = fingerprint.Value, true
}
}
if found != state.wantSet || (found && value != state.wantValue) {
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
RawInputDigest: "sha256:input",
RuntimeOverrides: fingerprints,
})
if err != nil {
t.Fatal(err)
}
digests[state.name] = identity.Digest
}
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
}
}
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
resolved := pipeline.ResolvedPipeline{
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},

10
internal/cli/spell_catalog_identity_contract_test.go
View File

@@ -216,7 +216,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
fingerprints := prepared.CheckpointFingerprints()
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", false)
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
if err != nil {
t.Fatal(err)
}
@@ -242,7 +242,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", true)
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
@@ -254,7 +254,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
}
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", true)
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
@@ -266,7 +266,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
}
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", true)
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
@@ -275,7 +275,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
}
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", true)
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}