From f8333f2c153439d074695ec48f14d3f2a1eabe1d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 02:11:35 +0000 Subject: [PATCH] Expose run-wide reasoning effort controls --- docs/cli.md | 13 +- docs/internal/cli.md | 10 +- docs/internal/llm.md | 7 +- docs/operations.md | 12 +- internal/cli/run.go | 69 +++++++--- internal/cli/run_contract_test.go | 124 ++++++++++++++++++ .../spell_catalog_identity_contract_test.go | 10 +- internal/core/debugbundle/bundle_test.go | 42 ++++++ internal/core/debugbundle/summary.go | 29 ++-- 9 files changed, 269 insertions(+), 47 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 40d0644..2ed830b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -41,14 +41,21 @@ pipeline ID and **--input** are required. | **--only lane-a,lane-b** | Run only the selected comma-separated artifact lanes when that selection is valid for the configured pipeline. | | **--llm-profile id** | Override effective LLM-capable module bindings with one configured profile. | | **--session-id id** | Supply a non-empty prompt session identifier to LLM-backed module calls. | +| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. | +| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. | | **--reference selector=path** | Add or replace a file reference binding. Repeatable. | | **--without-reference selector** | Remove a configured optional reference binding. Repeatable. | **--chunk_cache** accepts only **auto**, **bypass**, or **refresh**. **--debug-dir**, **--output-dir**, **--session-id**, and -**--recompute-step** reject explicit empty values. **--recompute-step** -requires **--resume**; checkpoint requirements and reuse behavior are -documented in [Operations](operations.md). +**--reasoning-effort**, and **--recompute-step** reject explicit empty values. +**--reasoning-effort** and **--clear-reasoning-effort** are mutually exclusive. +When neither is present, reasoning effort comes from the selected PromptKit +profile. These controls apply to the shared run client, including retries and +LLM-backed validators, and do not modify configuration or profile files. +Persistent reasoning settings remain a PromptKit profile concern. +**--recompute-step** requires **--resume**; checkpoint requirements and reuse +behavior are documented in [Operations](operations.md). ### Reference selectors diff --git a/docs/internal/cli.md b/docs/internal/cli.md index b45059b..e88850e 100644 --- a/docs/internal/cli.md +++ b/docs/internal/cli.md @@ -63,9 +63,13 @@ the adapter mechanics remain in [LLM Runtime](llm.md). The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer preserves inherit, replace, and clear states across the composition boundary. -Run orchestration currently passes the zero value, so production execution -inherits the selected PromptKit profile. No command flag or Notarius -configuration field exposes this internal override yet. +Run orchestration constructs this value from the mutually exclusive +`--reasoning-effort` and `--clear-reasoning-effort` controls. Absence preserves +a nil pointer, replacement is trimmed, and clear uses a non-nil empty string. +The same override reaches the one shared production client, checkpoint +identity, and debug invocation metadata. Persistent reasoning configuration +remains owned by PromptKit profiles; Notarius configuration has no reasoning +field. ## Run Orchestration diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 53e4d00..287337a 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -40,8 +40,11 @@ Client construction may also receive a run-wide reasoning-effort override from the CLI factory boundary. The adapter copies the caller-owned pointer and creates a fresh PromptKit execution override for each request: a nil pointer inherits the selected profile, a non-empty value replaces it, and an empty -value clears inherited reasoning. Ordinary CLI execution currently supplies no -override, so profile behavior remains unchanged. +value clears inherited reasoning. The CLI's mutually exclusive +`--reasoning-effort` and `--clear-reasoning-effort` controls select those +states. With neither flag, profile behavior remains unchanged. Because +production constructs one shared client, the selected state applies uniformly +to module calls, retries, and LLM-backed validators for the whole run. An empty request profile lets the prompt select its configured default. The CLI prepares every explicitly selected binding profile before a run begins, so a diff --git a/docs/operations.md b/docs/operations.md index 3e4f29f..077b62a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -121,6 +121,9 @@ provenance, the effective PromptKit profile-source fingerprint, and prepared-component fingerprints. Changing profile content causes a cold miss even when its profile ID is unchanged. A changed identity produces a cold miss; Notarius does not migrate, rewrite, or delete older checkpoint directories. +Reasoning-effort inheritance, replacement, and explicit clearing are distinct +runtime identities, so checkpoints created under one state are not reused by +either of the others. Checkpoint state is confined below an identity-specific path: @@ -180,7 +183,9 @@ warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace contains allowlisted application diagnostic records and can include source or derived application data. Neither surface is a cache input. Do not treat a debug bundle as safe to share merely because its configuration summary is -redacted. +redacted. Invocation metadata omits reasoning effort when it is inherited, +records the replacement value when one is supplied, and records an empty value +when inherited reasoning was explicitly cleared. Notarius never creates debug state without an explicit request and never automatically deletes a requested bundle. If allocation succeeds, the command @@ -209,7 +214,10 @@ or automatic cleanup command. ## Operational Limits Provider execution settings and the generation timeout come from the selected -PromptKit profile. PromptKit v0.1.0 does not add a provider retry loop; +PromptKit profile. The invocation-only **--reasoning-effort** and +**--clear-reasoning-effort** controls may replace or clear that profile setting +for all LLM-backed calls in one run without changing the profile. PromptKit +v0.2.0 does not add a provider retry loop; Notarius binding retries rerun the complete module operation and validation chain as defined by [module bindings](config.md#module-bindings-and-validators). diff --git a/internal/cli/run.go b/internal/cli/run.go index 22a3794..07e70ba 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -142,13 +142,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i debug := fs.Bool("debug", false, "write a debug bundle") debugDir := fs.String("debug-dir", "", "debug bundle directory") llmProfile := fs.String("llm-profile", "", "LLM profile override") + reasoningEffort := singleValueFlag{name: "--reasoning-effort"} + clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort") resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints") - recomputeStep := singleValueFlag{} + recomputeStep := singleValueFlag{name: "--recompute-step"} chunkCache := chunkCacheFlag{} sessionID := sessionIDFlag{} referenceFlags := stringListFlag{} withoutReferenceFlags := stringListFlag{} fs.Var(&sessionID, "session-id", "prompt session identifier") + fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override") fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh") fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path") fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference") @@ -194,6 +197,22 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i fmt.Fprintln(stderr, "notarius: --session-id must not be empty") return 2 } + if reasoningEffort.set && *clearReasoningEffort { + fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort") + return 2 + } + if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" { + fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty") + return 2 + } + runtimeOverrides := LLMRuntimeOverrides{} + if reasoningEffort.set { + value := strings.TrimSpace(reasoningEffort.value) + runtimeOverrides.ReasoningEffort = &value + } else if *clearReasoningEffort { + value := "" + runtimeOverrides.ReasoningEffort = &value + } only, err := parseOnly(*onlyRaw) if err != nil { fmt.Fprintf(stderr, "notarius: %v\n", err) @@ -284,17 +303,18 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder) } invocation := debugbundle.Invocation{ - Operation: "run", - PipelineID: pipelineID, - InputPath: strings.TrimSpace(*inputPath), - ConfigPath: loadedConfigPath, - ConfigSource: configSource(*configPath), - OnlyLanes: append([]string(nil), only...), - ChunkCacheOverride: chunkCache.explicitValue(), - Resume: *resume, - RecomputeStep: strings.TrimSpace(recomputeStep.value), - RunID: runID, - StartedAt: startedAt, + Operation: "run", + PipelineID: pipelineID, + InputPath: strings.TrimSpace(*inputPath), + ConfigPath: loadedConfigPath, + ConfigSource: configSource(*configPath), + OnlyLanes: append([]string(nil), only...), + ChunkCacheOverride: chunkCache.explicitValue(), + ReasoningEffortOverride: runtimeOverrides.ReasoningEffort, + Resume: *resume, + RecomputeStep: strings.TrimSpace(recomputeStep.value), + RunID: runID, + StartedAt: startedAt, } if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err)) @@ -368,7 +388,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if len(profileIDs) == 1 { factoryProfileID = profileIDs[0] } - llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, LLMRuntimeOverrides{}) + llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } @@ -392,7 +412,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } - checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) + checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), runtimeOverrides, *resume) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } @@ -499,6 +519,7 @@ func checkpointHandlersForRun( llmProfiles []artifacts.LLMProfileManifest, llmProfileOverride string, sessionID string, + runtimeOverrides LLMRuntimeOverrides, resume bool, ) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) { if !settings.Enabled { @@ -512,7 +533,7 @@ func checkpointHandlersForRun( InputKey: resolved.Input.Module, RawInputDigest: rawInputDigest(rawInput), SelectedLanes: only, - RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID), + RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides), References: pipeline.ReferenceProvenance(resolved), ProvenanceFingerprints: combineCheckpointFingerprints( llmProfileFingerprints(llmProfiles), @@ -673,7 +694,7 @@ func rawInputDigest(data []byte) string { return "sha256:" + hex.EncodeToString(sum[:]) } -func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []checkpoint.Fingerprint { +func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint { var values []checkpoint.Fingerprint if strings.TrimSpace(llmProfileOverride) != "" { values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)}) @@ -681,6 +702,13 @@ func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) [] if strings.TrimSpace(sessionID) != "" { values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)}) } + if runtimeOverrides.ReasoningEffort != nil { + value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort) + if value == "" { + value = "" + } + values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value}) + } return values } @@ -837,7 +865,7 @@ func reorderRunArgs(args []string) []string { func runFlagTakesValue(arg string) bool { switch arg { - case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--chunk_cache", "--reference", "--without-reference", "--recompute-step": + case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step": return true default: return false @@ -897,11 +925,11 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli func validateRunFlagValues(args []string) error { for i, arg := range args { - if arg != "--session-id" { + if arg != "--session-id" && arg != "--reasoning-effort" { continue } if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { - return fmt.Errorf("flag needs an argument: --session-id") + return fmt.Errorf("flag needs an argument: %s", arg) } } return nil @@ -1185,6 +1213,7 @@ type sessionIDFlag struct { } type singleValueFlag struct { + name string value string set bool } @@ -1198,7 +1227,7 @@ func (flag *singleValueFlag) String() string { func (flag *singleValueFlag) Set(value string) error { if flag.set { - return fmt.Errorf("--recompute-step may be specified only once") + return fmt.Errorf("%s may be specified only once", flag.name) } flag.value = value flag.set = true diff --git a/internal/cli/run_contract_test.go b/internal/cli/run_contract_test.go index b27bc07..c2109e5 100644 --- a/internal/cli/run_contract_test.go +++ b/internal/cli/run_contract_test.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) @@ -319,6 +320,129 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) { }) } +func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) { + tests := []struct { + name string + flags []string + wantValue string + wantSet bool + }{ + {name: "inherit"}, + {name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true}, + {name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + roots := newStateTestRoots(t) + opts := newStateTestHarness().options() + var got []LLMRuntimeOverrides + opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { + got = append(got, overrides) + return nil, nil, nil + } + args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...) + var stdout, stderr bytes.Buffer + if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if len(got) != 1 { + t.Fatalf("factory overrides = %#v, want one call", got) + } + if !tt.wantSet { + if got[0].ReasoningEffort != nil { + t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort) + } + return + } + if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue { + t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue) + } + }) + } +} + +func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) { + tests := []struct { + name string + flags []string + wantError string + }{ + { + name: "mutually exclusive controls", + flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"}, + wantError: "cannot be combined", + }, + { + name: "empty replacement", + flags: []string{"--reasoning-effort", " "}, + wantError: "must not be empty", + }, + { + name: "duplicate replacement", + flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"}, + wantError: "may be specified only once", + }, + { + name: "missing replacement", + flags: []string{"--reasoning-effort"}, + wantError: "flag needs an argument", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + roots := newStateTestRoots(t) + args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...) + var stdout, stderr bytes.Buffer + code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options()) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + } +} + +func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) { + replacement := " focused " + cleared := "" + states := []struct { + name string + overrides LLMRuntimeOverrides + wantValue string + wantSet bool + }{ + {name: "inherit"}, + {name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true}, + {name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "", wantSet: true}, + } + digests := make(map[string]string, len(states)) + for _, state := range states { + fingerprints := runtimeOverrideFingerprints("", "", state.overrides) + var value string + var found bool + for _, fingerprint := range fingerprints { + if fingerprint.Name == "reasoning_effort_override" { + value, found = fingerprint.Value, true + } + } + if found != state.wantSet || (found && value != state.wantValue) { + t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue) + } + identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ + Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")}, + RawInputDigest: "sha256:input", + RuntimeOverrides: fingerprints, + }) + if err != nil { + t.Fatal(err) + } + digests[state.name] = identity.Digest + } + if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] { + t.Fatalf("checkpoint identity digests are not distinct: %#v", digests) + } +} + func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) { resolved := pipeline.ResolvedPipeline{ Input: pipeline.ModuleBinding{LLMProfile: "input-profile"}, diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go index b5f88fc..0d4f09f 100644 --- a/internal/cli/spell_catalog_identity_contract_test.go +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -216,7 +216,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t fingerprints := prepared.CheckpointFingerprints() llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}} settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} - recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", false) + recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false) if err != nil { t.Fatal(err) } @@ -242,7 +242,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t t.Fatal(err) } - _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", true) + _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) if err != nil { t.Fatal(err) } @@ -254,7 +254,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t } changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) - _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", true) + _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) if err != nil { t.Fatal(err) } @@ -266,7 +266,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t } changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) - _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", true) + _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) if err != nil { t.Fatal(err) } @@ -275,7 +275,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t } changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}} - _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", true) + _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) if err != nil { t.Fatal(err) } diff --git a/internal/core/debugbundle/bundle_test.go b/internal/core/debugbundle/bundle_test.go index e830ae9..f4c0ce0 100644 --- a/internal/core/debugbundle/bundle_test.go +++ b/internal/core/debugbundle/bundle_test.go @@ -2,6 +2,7 @@ package debugbundle import ( "bytes" + "encoding/json" "os" "path/filepath" "strings" @@ -137,6 +138,47 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) { } } } + +func TestWriteInvocationPreservesReasoningEffortOverrideStates(t *testing.T) { + replacement := "focused" + cleared := "" + tests := []struct { + name string + override *string + wantValue string + wantSet bool + }{ + {name: "inherit"}, + {name: "replace", override: &replacement, wantValue: "focused", wantSet: true}, + {name: "clear", override: &cleared, wantSet: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42)) + if err != nil { + t.Fatal(err) + } + if err := bundle.Summary().WriteInvocation(Invocation{ + Operation: "run", + ReasoningEffortOverride: tt.override, + }); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(bundle.SummaryRoot(), ArtifactInvocationMetadata)) + if err != nil { + t.Fatal(err) + } + var payload map[string]any + if err := json.Unmarshal(data, &payload); err != nil { + t.Fatal(err) + } + value, found := payload["reasoning_effort_override"] + if found != tt.wantSet || (found && value != tt.wantValue) { + t.Fatalf("reasoning override found=%t value=%#v, want found=%t value=%q; JSON=%s", found, value, tt.wantSet, tt.wantValue, data) + } + }) + } +} func TestSummaryWriterInternalWritesConfineArtifacts(t *testing.T) { bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42)) if err != nil { diff --git a/internal/core/debugbundle/summary.go b/internal/core/debugbundle/summary.go index 6f957d2..c937b0b 100644 --- a/internal/core/debugbundle/summary.go +++ b/internal/core/debugbundle/summary.go @@ -29,18 +29,19 @@ type RedactedResolvedPipelinePayload interface { RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline } type Invocation struct { - Operation string `json:"operation"` - PipelineID string `json:"pipeline_id,omitempty"` - PipelineDigest string `json:"pipeline_digest,omitempty"` - Resume bool `json:"resume,omitempty"` - RecomputeStep string `json:"recompute_step,omitempty"` - InputPath string `json:"input_path,omitempty"` - ConfigPath string `json:"config_path,omitempty"` - ConfigSource string `json:"config_source,omitempty"` - OnlyLanes []string `json:"only_lanes,omitempty"` - ChunkCacheOverride string `json:"chunk_cache_override,omitempty"` - RunID string `json:"run_id"` - StartedAt time.Time `json:"started_at"` + Operation string `json:"operation"` + PipelineID string `json:"pipeline_id,omitempty"` + PipelineDigest string `json:"pipeline_digest,omitempty"` + Resume bool `json:"resume,omitempty"` + RecomputeStep string `json:"recompute_step,omitempty"` + InputPath string `json:"input_path,omitempty"` + ConfigPath string `json:"config_path,omitempty"` + ConfigSource string `json:"config_source,omitempty"` + OnlyLanes []string `json:"only_lanes,omitempty"` + ChunkCacheOverride string `json:"chunk_cache_override,omitempty"` + ReasoningEffortOverride *string `json:"reasoning_effort_override,omitempty"` + RunID string `json:"run_id"` + StartedAt time.Time `json:"started_at"` } type RunReport struct { RunID string `json:"run_id"` @@ -68,6 +69,10 @@ func (w *SummaryWriter) WriteInvocation(payload Invocation) error { if payload.StartedAt.IsZero() { payload.StartedAt = w.createdAt } + if payload.ReasoningEffortOverride != nil { + value := *payload.ReasoningEffortOverride + payload.ReasoningEffortOverride = &value + } return w.writeJSON(ArtifactInvocationMetadata, payload) } func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayload) error {