eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse valid workspace checkpoints on request

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 03:02:50 +00:00
parent 1d3a444df8
commit ae9c2e1d5e
14 changed files with 1406 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
internal/cli/run.go
View File

@@ -29,7 +29,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -99,6 +99,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
@@ -179,12 +180,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
Resume: *resume,
RunID: runID,
StartedAt: startedAt,
}
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
}
if *resume && !workspaceSettings.ResumeEnabled {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
}
catalog, err := effectiveCatalog(opts)
if err != nil {
@@ -256,7 +261,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
checkpointRecorder, err := checkpointRecorderForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value))
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
@@ -273,10 +278,12 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
Checkpoints: checkpointRecorder,
Checkpoint: checkpointLoader,
})
if err != nil {
if output.Manifest.PipelineID != "" && runDir != nil {
_ = runDir.WriteRunManifest(output.Manifest)
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
}
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
}
@@ -288,6 +295,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
}
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
}
if err := writeDiagnostics(runDir, func() error {
return runDir.WriteRunReport(runReport{
RunID: runDir.RunID(),
@@ -356,7 +366,7 @@ func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) erro
return write()
}
func checkpointRecorderForRun(
func checkpointHandlersForRun(
settings workspace.Settings,
resolved pipeline.ResolvedPipeline,
rawInput []byte,
@@ -364,7 +374,8 @@ func checkpointRecorderForRun(
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
) (pipeline.CheckpointRecorder, error) {
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
@@ -375,13 +386,20 @@ func checkpointRecorderForRun(
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
})
if err != nil {
return nil, fmt.Errorf("create checkpoint identity: %w", err)
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
}
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
if err != nil {
return nil, fmt.Errorf("create checkpoint recorder: %w", err)
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
}
return recorder, nil
loader := pipeline.NoopCheckpointLoader()
if resume {
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
}
}
return recorder, loader, nil
}
func rawInputDigest(data []byte) string {