eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse valid workspace checkpoints on request

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 03:02:50 +00:00
parent 1d3a444df8
commit ae9c2e1d5e
14 changed files with 1406 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
internal/cli/run.go
View File

@@ -29,7 +29,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -99,6 +99,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
@@ -179,12 +180,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
Resume: *resume,
RunID: runID,
StartedAt: startedAt,
}
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
}
if *resume && !workspaceSettings.ResumeEnabled {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
}
catalog, err := effectiveCatalog(opts)
if err != nil {
@@ -256,7 +261,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
checkpointRecorder, err := checkpointRecorderForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value))
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
@@ -273,10 +278,12 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
Checkpoints: checkpointRecorder,
Checkpoint: checkpointLoader,
})
if err != nil {
if output.Manifest.PipelineID != "" && runDir != nil {
_ = runDir.WriteRunManifest(output.Manifest)
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
}
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
}
@@ -288,6 +295,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
}
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
}
if err := writeDiagnostics(runDir, func() error {
return runDir.WriteRunReport(runReport{
RunID: runDir.RunID(),
@@ -356,7 +366,7 @@ func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) erro
return write()
}
func checkpointRecorderForRun(
func checkpointHandlersForRun(
settings workspace.Settings,
resolved pipeline.ResolvedPipeline,
rawInput []byte,
@@ -364,7 +374,8 @@ func checkpointRecorderForRun(
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
) (pipeline.CheckpointRecorder, error) {
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
@@ -375,13 +386,20 @@ func checkpointRecorderForRun(
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
})
if err != nil {
return nil, fmt.Errorf("create checkpoint identity: %w", err)
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
}
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
if err != nil {
return nil, fmt.Errorf("create checkpoint recorder: %w", err)
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
}
return recorder, nil
loader := pipeline.NoopCheckpointLoader()
if resume {
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
}
}
return recorder, loader, nil
}
func rawInputDigest(data []byte) string {

266
internal/cli/run_test.go
View File

@@ -2185,16 +2185,20 @@ func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
client := newFakeRunLLMClient(false)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want ordinary run to execute despite checkpoint writing", client.calls)
}
checkpointDir := onlyCheckpointIdentityDir(t, workspaceDir)
for _, name := range []string{
"source/manifest.json",
@@ -2215,6 +2219,133 @@ func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
}
func TestRunPipelineResumeRequiresWorkspaceResumeEnabled(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnostics("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "--resume requires workspace.resume.enabled: true") {
t.Fatalf("stderr = %q, want resume configuration error", stderr.String())
}
}
func TestRunPipelineResumeReusesWorkspaceCheckpoints(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
firstClient := newFakeRunLLMClient(false)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(firstClient, nil),
})
if code != 0 {
t.Fatalf("first RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if firstClient.calls != 1 {
t.Fatalf("first LLM calls = %d, want checkpoint seed run to execute", firstClient.calls)
}
secondClient := newFakeRunLLMClient(false)
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(secondClient, nil),
})
if code != 0 {
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if secondClient.calls != 0 {
t.Fatalf("resume LLM calls = %d, want checkpoint reuse", secondClient.calls)
}
runDirs := childDirs(t, filepath.Join(workspaceDir, "diagnostics"))
if len(runDirs) != 2 {
t.Fatalf("diagnostics run dirs = %v, want fresh diagnostics for each invocation", runDirs)
}
if !anyDiagnosticsFileContains(t, runDirs, diagnostics.ArtifactCheckpointEvents, `"action": "reused"`) {
t.Fatalf("checkpoint event diagnostics under %v did not record reuse", runDirs)
}
}
func TestRunPipelineResumeInvalidatesWhenInvocationIdentityChanges(t *testing.T) {
t.Run("input", func(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
seedInput := writeSeriatimInput(t)
changedInput := writeFile(t, "source-changed.json", `{
"metadata": {"id": "session-alpha"},
"segments": [{"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Shield."}]
}`)
seedWorkspaceCheckpoint(t, configPath, seedInput, nil)
client := runResumeWithClient(t, configPath, changedInput, nil)
if client.calls == 0 {
t.Fatal("resume LLM calls = 0, want execution after input identity change")
}
})
t.Run("pipeline digest", func(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
seedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
changedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndChunkOptions("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
seedWorkspaceCheckpoint(t, seedConfig, inputPath, nil)
client := runResumeWithClient(t, changedConfig, inputPath, nil)
if client.calls == 0 {
t.Fatal("resume LLM calls = 0, want execution after pipeline digest change")
}
})
t.Run("selected lanes", func(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeLanes("dnd-session", workspaceDir, "always", "spells", "items"))
inputPath := writeSeriatimInput(t)
seedWorkspaceCheckpoint(t, configPath, inputPath, nil)
client := runResumeWithClient(t, configPath, inputPath, []string{"--only", "spells"})
if client.calls == 0 {
t.Fatal("resume LLM calls = 0, want execution after selected lane change")
}
})
t.Run("references", func(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
referencePath := writeFile(t, "players.md", "Aria is a cleric.\n")
seedWorkspaceCheckpoint(t, configPath, inputPath, []string{"--reference", "players=" + referencePath})
if err := os.WriteFile(referencePath, []byte("Aria is a wizard.\n"), 0o644); err != nil {
t.Fatalf("update reference: %v", err)
}
client := runResumeWithClient(t, configPath, inputPath, []string{"--reference", "players=" + referencePath})
if client.calls == 0 {
t.Fatal("resume LLM calls = 0, want execution after reference digest change")
}
})
t.Run("llm profile override", func(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndProfileFile("dnd-session", workspaceDir, "always", profilePath))
inputPath := writeSeriatimInput(t)
seedWorkspaceCheckpoint(t, configPath, inputPath, nil)
client := runResumeWithClient(t, configPath, inputPath, []string{"--llm-profile", "runtime"})
if client.calls == 0 {
t.Fatal("resume LLM calls = 0, want execution after LLM profile override change")
}
})
}
func TestRunPipelineSkipsDiagnosticsWhenWorkspaceDiagnosticsDisabled(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
@@ -2923,6 +3054,88 @@ pipelines:
`
}
func mvpConfigYAMLWithWorkspaceResumeAndChunker(pipelineID, workspaceDir, retention, chunker string) string {
return `version: 2
workspace:
directory: ` + workspaceDir + `
diagnostics:
enabled: true
retention: ` + retention + `
resume:
enabled: true
pipelines:
` + pipelineID + `:
input: seriatim
chunk: ` + chunker + `
artifacts:
spells:
extract: dnd/spells
`
}
func mvpConfigYAMLWithWorkspaceResumeAndChunkOptions(pipelineID, workspaceDir, retention string) string {
return `version: 2
workspace:
directory: ` + workspaceDir + `
diagnostics:
enabled: true
retention: ` + retention + `
resume:
enabled: true
pipelines:
` + pipelineID + `:
input: seriatim
chunk:
module: generic
options:
max_units: 10
artifacts:
spells:
extract: dnd/spells
`
}
func mvpConfigYAMLWithWorkspaceResumeAndProfileFile(pipelineID, workspaceDir, retention, profileFile string) string {
return `version: 2
scriptorium:
profile_file: ` + profileFile + `
workspace:
directory: ` + workspaceDir + `
diagnostics:
enabled: true
retention: ` + retention + `
resume:
enabled: true
pipelines:
` + pipelineID + `:
input: seriatim
artifacts:
spells:
extract: dnd/spells
`
}
func mvpConfigYAMLWithWorkspaceResumeLanes(pipelineID, workspaceDir, retention string, laneIDs ...string) string {
var b strings.Builder
b.WriteString("version: 2\n")
b.WriteString("workspace:\n")
b.WriteString(" directory: " + workspaceDir + "\n")
b.WriteString(" diagnostics:\n")
b.WriteString(" enabled: true\n")
b.WriteString(" retention: " + retention + "\n")
b.WriteString(" resume:\n")
b.WriteString(" enabled: true\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: seriatim\n")
b.WriteString(" artifacts:\n")
for _, laneID := range laneIDs {
b.WriteString(" " + laneID + ":\n")
b.WriteString(" extract: dnd/spells\n")
}
return b.String()
}
func mvpConfigYAMLWithWorkspaceDiagnosticsDisabled(pipelineID, workspaceDir string) string {
return `version: 2
workspace:
@@ -3293,6 +3506,57 @@ func onlyCheckpointIdentityDir(t *testing.T, workspaceDir string) string {
return onlyChildDir(t, inputDir)
}
func anyDiagnosticsFileContains(t *testing.T, runDirs []string, name string, want string) bool {
t.Helper()
for _, runDir := range runDirs {
data, err := os.ReadFile(filepath.Join(runDir, name))
if err != nil {
if os.IsNotExist(err) {
continue
}
t.Fatalf("read diagnostics artifact %q under %q: %v", name, runDir, err)
}
if strings.Contains(string(data), want) {
return true
}
}
return false
}
func seedWorkspaceCheckpoint(t *testing.T, configPath string, inputPath string, extraArgs []string) {
t.Helper()
client := newFakeRunLLMClient(false)
args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}
args = append(args, extraArgs...)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if client.calls == 0 {
t.Fatal("seed LLM calls = 0, want checkpoint seed run to execute")
}
}
func runResumeWithClient(t *testing.T, configPath string, inputPath string, extraArgs []string) *fakeRunLLMClient {
t.Helper()
client := newFakeRunLLMClient(false)
args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir(), "--resume"}
args = append(args, extraArgs...)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
return client
}
func childDirs(t *testing.T, root string) []string {
t.Helper()
entries, err := os.ReadDir(root)