Execute ordered pipeline steps with barriers

internal/framework/pipeline/prepare.go

@@ -12,21 +12,26 @@ import (
 // resolved pipeline. Its implementation values are private so execution cannot
 // replace or reconfigure them after preparation.
 type PreparedPipeline struct {
-	Input         ModuleBinding
-	Chunk         ModuleBinding
-	ArtifactLanes []PreparedArtifactLane
-	Output        ModuleBinding
+	Input  ModuleBinding
+	Chunk  ModuleBinding
+	Steps  []PreparedPipelineStep
+	Output ModuleBinding
 
 	resolved               ResolvedPipeline
 	dependencies           ModuleDependencies
 	input                  contracts.InputAdapter
 	chunker                contracts.Chunker
 	chunkValidators        preparedValidatorChain
-	lanes                  []preparedLaneExecutor
 	output                 contracts.OutputEncoder
 	checkpointFingerprints []CheckpointFingerprint
 }
 
+type PreparedPipelineStep struct {
+	ID            string
+	ArtifactLanes []PreparedArtifactLane
+	lanes         []preparedLaneExecutor
+}
+
 type PreparedArtifactLane struct {
 	Resolved ResolvedArtifactLane
 }
@@ -68,9 +73,6 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
 	if err := validateResolvedPipeline(resolved); err != nil {
 		return nil, err
 	}
-	if len(resolved.Steps) > 1 {
-		return nil, fmt.Errorf("pipeline %q contains multiple ordered steps; execution support is not available yet", resolved.ID)
-	}
 	if err := validateRegistrySet(resolved, registries); err != nil {
 		return nil, err
 	}
@@ -102,16 +104,22 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
 		return nil, err
 	}
 
-	lanes := stable.AllArtifactLanes()
-	prepared.ArtifactLanes = make([]PreparedArtifactLane, 0, len(lanes))
-	prepared.lanes = make([]preparedLaneExecutor, 0, len(lanes))
-	for _, lane := range lanes {
-		executor, err := prepareLane(stable, lane, registries, deps)
-		if err != nil {
-			return nil, err
+	prepared.Steps = make([]PreparedPipelineStep, len(stable.Steps))
+	for stepIndex, step := range stable.Steps {
+		preparedStep := PreparedPipelineStep{
+			ID:            step.ID,
+			ArtifactLanes: make([]PreparedArtifactLane, 0, len(step.ArtifactLanes)),
+			lanes:         make([]preparedLaneExecutor, 0, len(step.ArtifactLanes)),
 		}
-		prepared.ArtifactLanes = append(prepared.ArtifactLanes, PreparedArtifactLane{Resolved: cloneResolvedArtifactLane(lane)})
-		prepared.lanes = append(prepared.lanes, executor)
+		for _, lane := range step.ArtifactLanes {
+			executor, err := prepareLane(stable, lane, registries, deps)
+			if err != nil {
+				return nil, err
+			}
+			preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{Resolved: cloneResolvedArtifactLane(lane)})
+			preparedStep.lanes = append(preparedStep.lanes, executor)
+		}
+		prepared.Steps[stepIndex] = preparedStep
 	}
 
 	output, err := registries.Outputs.BuildWithRequest(stable.Output.Module, request(stable.Output, contracts.ReferenceSet{}))
@@ -126,6 +134,31 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
 	return prepared, nil
 }
 
+func rejectGeneratedBindings(resolved ResolvedPipeline) error {
+	check := func(stepID, laneID string, target ResolvedReferenceTarget) error {
+		for _, binding := range target.Bindings {
+			if binding.Artifact == nil {
+				continue
+			}
+			return fmt.Errorf("pipeline %q step %q lane %q %s reference slot %q uses a generated artifact; generated reference execution is not supported yet", resolved.ID, stepID, laneID, target.Stage, binding.SlotName)
+		}
+		return nil
+	}
+	if err := check("", "", resolved.ChunkReferences); err != nil {
+		return err
+	}
+	for _, step := range resolved.Steps {
+		for _, lane := range step.ArtifactLanes {
+			for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
+				if err := check(step.ID, lane.ID, target); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
 	executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
 	request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {