eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decompose pipeline runner orchestration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-22 02:03:27 +00:00
parent 64ea23c21f
commit 5bdd56cfb1
4 changed files with 146 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/framework/pipeline/runner.go
View File

@@ -252,22 +252,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
if chunkResult.accepted {
for _, step := range input.Prepared.Steps {
stepInput := input
stepInput.stepID = step.ID
stepReferences, referenceProvenance, handoffErr := buildStepReferenceSets(input, step, output.NormalizeOutputs)
if handoffErr != nil {
return failOutput(output), fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, handoffErr)
}
stepInput.references = stepReferences
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks)
if err := mergeLaneOutput(&output, laneOutput); err != nil {
return failOutput(output), err
}
if laneErr != nil {
return failOutput(output), fmt.Errorf("execute pipeline step %q: %w", step.ID, laneErr)
}
if err := r.runPreparedSteps(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
return failOutput(output), err
}
}
@@ -335,6 +321,27 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
for _, step := range input.Prepared.Steps {
stepInput := input
stepInput.stepID = step.ID
stepReferences, referenceProvenance, err := buildStepReferenceSets(input, step, output.NormalizeOutputs)
if err != nil {
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
}
stepInput.references = stepReferences
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
if err := mergeLaneOutput(output, laneOutput); err != nil {
return err
}
if laneErr != nil {
return fmt.Errorf("execute pipeline step %q: %w", step.ID, laneErr)
}
}
return nil
}
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
attempts := retries + 1
var last *contracts.RejectedOutput