eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement ordered pipeline step resolution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-21 21:05:28 +00:00
parent f5618d1f0c
commit f846f252c0
44 changed files with 1335 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
internal/framework/pipeline/references.go
View File

@@ -36,37 +36,51 @@ func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, opt
}
out.ChunkReferences.ReferenceSet = chunkReferenceSet
warnings := append([]contracts.Warning(nil), chunkWarnings...)
if len(resolved.ArtifactLanes) == 0 {
if len(resolved.Steps) == 0 {
return out, warnings, nil
}
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializeLane := func(lane ResolvedArtifactLane) (ResolvedArtifactLane, []contracts.Warning, error) {
materializedLane := lane
var allWarnings []contracts.Warning
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.MergeReferences = CloneReferenceTarget(lane.MergeReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, lane.ArtifactKind, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
return ResolvedArtifactLane{}, nil, err
}
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
warnings = append(warnings, laneWarnings...)
allWarnings = append(allWarnings, laneWarnings...)
mergeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.MergeReferences, lane.ArtifactKind, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
return ResolvedArtifactLane{}, nil, err
}
materializedLane.MergeReferences.ReferenceSet = mergeReferenceSet
warnings = append(warnings, laneWarnings...)
allWarnings = append(allWarnings, laneWarnings...)
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, lane.ArtifactKind, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
return ResolvedArtifactLane{}, nil, err
}
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
warnings = append(warnings, laneWarnings...)
out.ArtifactLanes[i] = materializedLane
allWarnings = append(allWarnings, laneWarnings...)
return materializedLane, allWarnings, nil
}
if len(resolved.Steps) > 0 {
out.Steps = make([]ResolvedPipelineStep, len(resolved.Steps))
for i, step := range resolved.Steps {
out.Steps[i].ID = step.ID
out.Steps[i].ArtifactLanes = make([]ResolvedArtifactLane, len(step.ArtifactLanes))
for j, lane := range step.ArtifactLanes {
materializedLane, laneWarnings, err := materializeLane(lane)
if err != nil {
return ResolvedPipeline{}, nil, err
}
out.Steps[i].ArtifactLanes[j] = materializedLane
warnings = append(warnings, laneWarnings...)
}
}
}
return out, warnings, nil
}
@@ -99,6 +113,11 @@ func materializeReferenceTarget(
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
if binding.Artifact != nil {
// Generated selectors are resolved at the step handoff. They do not
// have bytes during static reference materialization.
continue
}
path, err := referencePath(binding, options)
if err != nil {
@@ -273,6 +292,7 @@ func fileURI(path string) string {
func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
slot.AcceptedArtifactKinds = append([]contracts.ArtifactKind(nil), slot.AcceptedArtifactKinds...)
return slot
}
@@ -304,7 +324,16 @@ func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
if len(in.Bindings) > 0 {
out.Bindings = make([]ReferenceBinding, len(in.Bindings))
for i, binding := range in.Bindings {
out.Bindings[i] = binding
if binding.Artifact != nil {
artifact := *binding.Artifact
out.Bindings[i].Artifact = &artifact
}
}
}
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
@@ -312,10 +341,14 @@ func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.MergeReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
if len(resolved.Steps) > 0 {
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.MergeReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
}
}
return provenance
}
@@ -335,6 +368,7 @@ func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.Refer
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
StepID: target.StepID,
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,