eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decompose pipeline runner orchestration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-22 02:03:27 +00:00
parent 64ea23c21f
commit 5bdd56cfb1
4 changed files with 146 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
internal/framework/pipeline/runner_concurrent.go
View File

@@ -77,28 +77,46 @@ type orderedRunError struct {
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
output := RunOutput{Manifest: manifestFromPipeline(input)}
states, err := initializeLaneStates(input, step, checkpoints, loader, doc, chunks, &output)
if err != nil {
return output, err
}
completedOutputs, runErrors := r.runLaneEngine(parent, input, checkpoints, loader, doc, sourceInput, sessionID, chunks, states)
if err := mergeCompletedLanes(&output, completedOutputs); err != nil {
return output, err
}
if err := selectRunError(parent, runErrors); err != nil {
return output, err
}
return output, nil
}
func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
states := make([]*laneExtractState, len(step.lanes))
for i, prepared := range step.lanes {
if prepared.typed == nil {
return output, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
return nil, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
}
if err := setTypedLaneManifestMetadata(&output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return output, err
if err := setTypedLaneManifestMetadata(output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return nil, err
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared)
if err != nil {
return output, err
return nil, err
}
if !state.decision.Reused {
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return output, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
return nil, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
} else if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
return output, err
return nil, err
}
states[i] = state
}
return states, nil
}
func (r *Runner) runLaneEngine(parent context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, states []*laneExtractState) ([]RunOutput, []orderedRunError) {
workerCount := input.ExtractWorkers
if workerCount < 1 {
workerCount = 1
@@ -216,15 +234,16 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedP
}
close(continuations)
continuationWorkers.Wait()
return completedOutputs, runErrors
}
func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error {
for i := range completedOutputs {
if err := mergeLaneOutput(&output, completedOutputs[i]); err != nil {
return output, err
if err := mergeLaneOutput(output, completedOutputs[i]); err != nil {
return err
}
}
if err := selectRunError(parent, runErrors); err != nil {
return output, err
}
return output, nil
return nil
}
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor) (*laneExtractState, error) {
@@ -237,19 +256,8 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
decision = forceCheckpointDecision(input.CheckpointPolicy, input.stepID, lane.ID, decision)
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, decision); err != nil {
return nil, err
}
if decision.Reused {
for _, stored := range cp.Outputs {
if _, _, decodeErr := decodeCanonicalCheckpointArtifact(typed.codec, stored); decodeErr != nil {
decision = CheckpointDecision{Category: "executed", ReasonCode: "artifact_not_canonical", Detail: "stored extract artifact failed canonical codec validation", Reason: "extract artifact checkpoint is not canonical"}
break
}
}
}
if err := requireReusableCheckpoint(input.CheckpointPolicy, input.stepID, lane.ID, decision); err != nil {
decision, err = resolveCheckpointDecision(nil, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
if err != nil {
return nil, err
}
state.decision = decision