Expose run-wide reasoning effort controls

This commit is contained in:
2026-07-30 02:11:35 +00:00
parent f603f7ac64
commit f8333f2c15
9 changed files with 269 additions and 47 deletions

View File

@@ -41,14 +41,21 @@ pipeline ID and **--input** are required.
| **--only lane-a,lane-b** | Run only the selected comma-separated artifact lanes when that selection is valid for the configured pipeline. | | **--only lane-a,lane-b** | Run only the selected comma-separated artifact lanes when that selection is valid for the configured pipeline. |
| **--llm-profile id** | Override effective LLM-capable module bindings with one configured profile. | | **--llm-profile id** | Override effective LLM-capable module bindings with one configured profile. |
| **--session-id id** | Supply a non-empty prompt session identifier to LLM-backed module calls. | | **--session-id id** | Supply a non-empty prompt session identifier to LLM-backed module calls. |
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. | | **--reference selector=path** | Add or replace a file reference binding. Repeatable. |
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. | | **--without-reference selector** | Remove a configured optional reference binding. Repeatable. |
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**. **--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
**--debug-dir**, **--output-dir**, **--session-id**, and **--debug-dir**, **--output-dir**, **--session-id**, and
**--recompute-step** reject explicit empty values. **--recompute-step** **--reasoning-effort**, and **--recompute-step** reject explicit empty values.
requires **--resume**; checkpoint requirements and reuse behavior are **--reasoning-effort** and **--clear-reasoning-effort** are mutually exclusive.
documented in [Operations](operations.md). When neither is present, reasoning effort comes from the selected PromptKit
profile. These controls apply to the shared run client, including retries and
LLM-backed validators, and do not modify configuration or profile files.
Persistent reasoning settings remain a PromptKit profile concern.
**--recompute-step** requires **--resume**; checkpoint requirements and reuse
behavior are documented in [Operations](operations.md).
### Reference selectors ### Reference selectors

View File

@@ -63,9 +63,13 @@ the adapter mechanics remain in [LLM Runtime](llm.md).
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
preserves inherit, replace, and clear states across the composition boundary. preserves inherit, replace, and clear states across the composition boundary.
Run orchestration currently passes the zero value, so production execution Run orchestration constructs this value from the mutually exclusive
inherits the selected PromptKit profile. No command flag or Notarius `--reasoning-effort` and `--clear-reasoning-effort` controls. Absence preserves
configuration field exposes this internal override yet. a nil pointer, replacement is trimmed, and clear uses a non-nil empty string.
The same override reaches the one shared production client, checkpoint
identity, and debug invocation metadata. Persistent reasoning configuration
remains owned by PromptKit profiles; Notarius configuration has no reasoning
field.
## Run Orchestration ## Run Orchestration

View File

@@ -40,8 +40,11 @@ Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer creates a fresh PromptKit execution override for each request: a nil pointer
inherits the selected profile, a non-empty value replaces it, and an empty inherits the selected profile, a non-empty value replaces it, and an empty
value clears inherited reasoning. Ordinary CLI execution currently supplies no value clears inherited reasoning. The CLI's mutually exclusive
override, so profile behavior remains unchanged. `--reasoning-effort` and `--clear-reasoning-effort` controls select those
states. With neither flag, profile behavior remains unchanged. Because
production constructs one shared client, the selected state applies uniformly
to module calls, retries, and LLM-backed validators for the whole run.
An empty request profile lets the prompt select its configured default. The CLI An empty request profile lets the prompt select its configured default. The CLI
prepares every explicitly selected binding profile before a run begins, so a prepares every explicitly selected binding profile before a run begins, so a

View File

@@ -121,6 +121,9 @@ provenance, the effective PromptKit profile-source fingerprint, and
prepared-component fingerprints. Changing profile content causes a cold miss prepared-component fingerprints. Changing profile content causes a cold miss
even when its profile ID is unchanged. A changed identity produces a cold miss; even when its profile ID is unchanged. A changed identity produces a cold miss;
Notarius does not migrate, rewrite, or delete older checkpoint directories. Notarius does not migrate, rewrite, or delete older checkpoint directories.
Reasoning-effort inheritance, replacement, and explicit clearing are distinct
runtime identities, so checkpoints created under one state are not reused by
either of the others.
Checkpoint state is confined below an identity-specific path: Checkpoint state is confined below an identity-specific path:
@@ -180,7 +183,9 @@ warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
contains allowlisted application diagnostic records and can include source or contains allowlisted application diagnostic records and can include source or
derived application data. Neither surface is a cache input. Do not treat a derived application data. Neither surface is a cache input. Do not treat a
debug bundle as safe to share merely because its configuration summary is debug bundle as safe to share merely because its configuration summary is
redacted. redacted. Invocation metadata omits reasoning effort when it is inherited,
records the replacement value when one is supplied, and records an empty value
when inherited reasoning was explicitly cleared.
Notarius never creates debug state without an explicit request and never Notarius never creates debug state without an explicit request and never
automatically deletes a requested bundle. If allocation succeeds, the command automatically deletes a requested bundle. If allocation succeeds, the command
@@ -209,7 +214,10 @@ or automatic cleanup command.
## Operational Limits ## Operational Limits
Provider execution settings and the generation timeout come from the selected Provider execution settings and the generation timeout come from the selected
PromptKit profile. PromptKit v0.1.0 does not add a provider retry loop; PromptKit profile. The invocation-only **--reasoning-effort** and
**--clear-reasoning-effort** controls may replace or clear that profile setting
for all LLM-backed calls in one run without changing the profile. PromptKit
v0.2.0 does not add a provider retry loop;
Notarius binding retries rerun the complete module operation and validation Notarius binding retries rerun the complete module operation and validation
chain as defined by [module bindings](config.md#module-bindings-and-validators). chain as defined by [module bindings](config.md#module-bindings-and-validators).

View File

@@ -142,13 +142,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debug := fs.Bool("debug", false, "write a debug bundle") debug := fs.Bool("debug", false, "write a debug bundle")
debugDir := fs.String("debug-dir", "", "debug bundle directory") debugDir := fs.String("debug-dir", "", "debug bundle directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override") llmProfile := fs.String("llm-profile", "", "LLM profile override")
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints") resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
recomputeStep := singleValueFlag{} recomputeStep := singleValueFlag{name: "--recompute-step"}
chunkCache := chunkCacheFlag{} chunkCache := chunkCacheFlag{}
sessionID := sessionIDFlag{} sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{} referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{} withoutReferenceFlags := stringListFlag{}
fs.Var(&sessionID, "session-id", "prompt session identifier") fs.Var(&sessionID, "session-id", "prompt session identifier")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh") fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
@@ -194,6 +197,22 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintln(stderr, "notarius: --session-id must not be empty") fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
return 2 return 2
} }
if reasoningEffort.set && *clearReasoningEffort {
fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
return 2
}
if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" {
fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty")
return 2
}
runtimeOverrides := LLMRuntimeOverrides{}
if reasoningEffort.set {
value := strings.TrimSpace(reasoningEffort.value)
runtimeOverrides.ReasoningEffort = &value
} else if *clearReasoningEffort {
value := ""
runtimeOverrides.ReasoningEffort = &value
}
only, err := parseOnly(*onlyRaw) only, err := parseOnly(*onlyRaw)
if err != nil { if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -291,6 +310,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigSource: configSource(*configPath), ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...), OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(), ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
Resume: *resume, Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value), RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID, RunID: runID,
@@ -368,7 +388,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if len(profileIDs) == 1 { if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0] factoryProfileID = profileIDs[0]
} }
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{}) llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
if err != nil { if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
} }
@@ -392,7 +412,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil { if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err) return failPipelineCommand(stderr, commandState, terminalWriter, err)
} }
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), runtimeOverrides, *resume)
if err != nil { if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err) return failPipelineCommand(stderr, commandState, terminalWriter, err)
} }
@@ -499,6 +519,7 @@ func checkpointHandlersForRun(
llmProfiles []artifacts.LLMProfileManifest, llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string, llmProfileOverride string,
sessionID string, sessionID string,
runtimeOverrides LLMRuntimeOverrides,
resume bool, resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) { ) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
if !settings.Enabled { if !settings.Enabled {
@@ -512,7 +533,7 @@ func checkpointHandlersForRun(
InputKey: resolved.Input.Module, InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput), RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only, SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID), RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
References: pipeline.ReferenceProvenance(resolved), References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: combineCheckpointFingerprints( ProvenanceFingerprints: combineCheckpointFingerprints(
llmProfileFingerprints(llmProfiles), llmProfileFingerprints(llmProfiles),
@@ -673,7 +694,7 @@ func rawInputDigest(data []byte) string {
return "sha256:" + hex.EncodeToString(sum[:]) return "sha256:" + hex.EncodeToString(sum[:])
} }
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []checkpoint.Fingerprint { func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
var values []checkpoint.Fingerprint var values []checkpoint.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" { if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)}) values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
@@ -681,6 +702,13 @@ func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []
if strings.TrimSpace(sessionID) != "" { if strings.TrimSpace(sessionID) != "" {
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)}) values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
} }
if runtimeOverrides.ReasoningEffort != nil {
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
if value == "" {
value = "<cleared>"
}
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
}
return values return values
} }
@@ -837,7 +865,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool { func runFlagTakesValue(arg string) bool {
switch arg { switch arg {
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--chunk_cache", "--reference", "--without-reference", "--recompute-step": case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
return true return true
default: default:
return false return false
@@ -897,11 +925,11 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli
func validateRunFlagValues(args []string) error { func validateRunFlagValues(args []string) error {
for i, arg := range args { for i, arg := range args {
if arg != "--session-id" { if arg != "--session-id" && arg != "--reasoning-effort" {
continue continue
} }
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
return fmt.Errorf("flag needs an argument: --session-id") return fmt.Errorf("flag needs an argument: %s", arg)
} }
} }
return nil return nil
@@ -1185,6 +1213,7 @@ type sessionIDFlag struct {
} }
type singleValueFlag struct { type singleValueFlag struct {
name string
value string value string
set bool set bool
} }
@@ -1198,7 +1227,7 @@ func (flag *singleValueFlag) String() string {
func (flag *singleValueFlag) Set(value string) error { func (flag *singleValueFlag) Set(value string) error {
if flag.set { if flag.set {
return fmt.Errorf("--recompute-step may be specified only once") return fmt.Errorf("%s may be specified only once", flag.name)
} }
flag.value = value flag.value = value
flag.set = true flag.set = true

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -319,6 +320,129 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
}) })
} }
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
tests := []struct {
name string
flags []string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
var got []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
got = append(got, overrides)
return nil, nil, nil
}
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(got) != 1 {
t.Fatalf("factory overrides = %#v, want one call", got)
}
if !tt.wantSet {
if got[0].ReasoningEffort != nil {
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
}
return
}
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
}
})
}
}
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
tests := []struct {
name string
flags []string
wantError string
}{
{
name: "mutually exclusive controls",
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
wantError: "cannot be combined",
},
{
name: "empty replacement",
flags: []string{"--reasoning-effort", " "},
wantError: "must not be empty",
},
{
name: "duplicate replacement",
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
wantError: "may be specified only once",
},
{
name: "missing replacement",
flags: []string{"--reasoning-effort"},
wantError: "flag needs an argument",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
}
}
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
replacement := " focused "
cleared := ""
states := []struct {
name string
overrides LLMRuntimeOverrides
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
}
digests := make(map[string]string, len(states))
for _, state := range states {
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
var value string
var found bool
for _, fingerprint := range fingerprints {
if fingerprint.Name == "reasoning_effort_override" {
value, found = fingerprint.Value, true
}
}
if found != state.wantSet || (found && value != state.wantValue) {
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
RawInputDigest: "sha256:input",
RuntimeOverrides: fingerprints,
})
if err != nil {
t.Fatal(err)
}
digests[state.name] = identity.Digest
}
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
}
}
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) { func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
resolved := pipeline.ResolvedPipeline{ resolved := pipeline.ResolvedPipeline{
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"}, Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},

View File

@@ -216,7 +216,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
fingerprints := prepared.CheckpointFingerprints() fingerprints := prepared.CheckpointFingerprints()
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}} llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", false) recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -242,7 +242,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatal(err) t.Fatal(err)
} }
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", true) _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -254,7 +254,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
} }
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", true) _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -266,7 +266,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
} }
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3") changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", true) _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -275,7 +275,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
} }
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}} changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", true) _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -2,6 +2,7 @@ package debugbundle
import ( import (
"bytes" "bytes"
"encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -137,6 +138,47 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
} }
} }
} }
func TestWriteInvocationPreservesReasoningEffortOverrideStates(t *testing.T) {
replacement := "focused"
cleared := ""
tests := []struct {
name string
override *string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", override: &replacement, wantValue: "focused", wantSet: true},
{name: "clear", override: &cleared, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42))
if err != nil {
t.Fatal(err)
}
if err := bundle.Summary().WriteInvocation(Invocation{
Operation: "run",
ReasoningEffortOverride: tt.override,
}); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(bundle.SummaryRoot(), ArtifactInvocationMetadata))
if err != nil {
t.Fatal(err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatal(err)
}
value, found := payload["reasoning_effort_override"]
if found != tt.wantSet || (found && value != tt.wantValue) {
t.Fatalf("reasoning override found=%t value=%#v, want found=%t value=%q; JSON=%s", found, value, tt.wantSet, tt.wantValue, data)
}
})
}
}
func TestSummaryWriterInternalWritesConfineArtifacts(t *testing.T) { func TestSummaryWriterInternalWritesConfineArtifacts(t *testing.T) {
bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42)) bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42))
if err != nil { if err != nil {

View File

@@ -39,6 +39,7 @@ type Invocation struct {
ConfigSource string `json:"config_source,omitempty"` ConfigSource string `json:"config_source,omitempty"`
OnlyLanes []string `json:"only_lanes,omitempty"` OnlyLanes []string `json:"only_lanes,omitempty"`
ChunkCacheOverride string `json:"chunk_cache_override,omitempty"` ChunkCacheOverride string `json:"chunk_cache_override,omitempty"`
ReasoningEffortOverride *string `json:"reasoning_effort_override,omitempty"`
RunID string `json:"run_id"` RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"` StartedAt time.Time `json:"started_at"`
} }
@@ -68,6 +69,10 @@ func (w *SummaryWriter) WriteInvocation(payload Invocation) error {
if payload.StartedAt.IsZero() { if payload.StartedAt.IsZero() {
payload.StartedAt = w.createdAt payload.StartedAt = w.createdAt
} }
if payload.ReasoningEffortOverride != nil {
value := *payload.ReasoningEffortOverride
payload.ReasoningEffortOverride = &value
}
return w.writeJSON(ArtifactInvocationMetadata, payload) return w.writeJSON(ArtifactInvocationMetadata, payload)
} }
func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayload) error { func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayload) error {