eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement ordered pipeline step resolution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-21 21:05:28 +00:00
parent f5618d1f0c
commit f846f252c0
44 changed files with 1335 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

381
internal/framework/pipeline/profile.go
View File

@@ -21,12 +21,56 @@ const (
)
type ModuleBinding struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators ValidatorOverride `json:"validators,omitempty"`
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
Validators ValidatorOverride `json:"validators,omitempty"`
}
// ArtifactReference identifies a normalized artifact produced by an earlier
// ordered step. It is a selector only; it does not contain artifact bytes.
type ArtifactReference struct {
Step string `json:"step"`
Lane string `json:"lane"`
}
// ReferenceSource is the discriminated source form used by configured
// reference maps. Exactly one of Path and Artifact is set after resolution.
type ReferenceSource struct {
Path string `json:"path,omitempty"`
Artifact *ArtifactReference `json:"artifact,omitempty"`
}
func ExternalReference(path string) ReferenceSource {
return ReferenceSource{Path: strings.TrimSpace(path)}
}
func ExternalReferenceMap(values map[string]string) map[string]ReferenceSource {
out := make(map[string]ReferenceSource, len(values))
for key, value := range values {
out[key] = ExternalReference(value)
}
return out
}
func GeneratedReference(step, lane string) ReferenceSource {
return ReferenceSource{Artifact: &ArtifactReference{Step: strings.TrimSpace(step), Lane: strings.TrimSpace(lane)}}
}
func (source ReferenceSource) IsGenerated() bool { return source.Artifact != nil }
func (source ReferenceSource) MarshalJSON() ([]byte, error) {
if source.Path != "" && source.Artifact != nil {
return nil, fmt.Errorf("reference source must not contain both an external path and an artifact selector")
}
if source.Artifact != nil {
return json.Marshal(struct {
Artifact *ArtifactReference `json:"artifact"`
}{Artifact: source.Artifact})
}
return json.Marshal(source.Path)
}
type ValidatorOverride struct {
@@ -36,12 +80,12 @@ type ValidatorOverride struct {
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
type moduleBindingJSON struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators *[]ModuleBinding `json:"validators,omitempty"`
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
Validators *[]ModuleBinding `json:"validators,omitempty"`
}
out := moduleBindingJSON{
Module: binding.Module,
@@ -58,11 +102,17 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
}
type ArtifactLaneProfile struct {
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
}
type PipelineStepProfile struct {
ID string `json:"id"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
References map[string]ReferenceSource `json:"references,omitempty"`
}
type PipelineProfile struct {
@@ -70,8 +120,9 @@ type PipelineProfile struct {
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Steps []PipelineStepProfile `json:"steps,omitempty"`
Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
}
type ResolveOptions struct {
@@ -81,11 +132,12 @@ type ResolveOptions struct {
}
type ReferenceBinding struct {
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source,omitempty"`
Artifact *ArtifactReference `json:"artifact,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
@@ -96,6 +148,7 @@ type ReferenceUnbind struct {
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
@@ -103,6 +156,7 @@ type ResolvedReferenceTarget struct {
}
type ResolvedArtifactLane struct {
StepID string
ID string
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
@@ -118,6 +172,11 @@ type ResolvedArtifactLane struct {
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipelineStep struct {
ID string `json:"id"`
ArtifactLanes []ResolvedArtifactLane `json:"artifact_lanes"`
}
type ResolvedValidatorChain struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
@@ -138,11 +197,21 @@ type ResolvedPipeline struct {
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding
}
// AllArtifactLanes returns lanes in deterministic step order for read-only
// discovery and identity construction.
func (resolved ResolvedPipeline) AllArtifactLanes() []ResolvedArtifactLane {
var lanes []ResolvedArtifactLane
for _, step := range resolved.Steps {
lanes = append(lanes, step.ArtifactLanes...)
}
return lanes
}
type ModuleCatalog struct {
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
@@ -165,7 +234,21 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return ResolvedPipeline{}, fmt.Errorf("pipeline id must not be empty")
}
if len(profile.Artifacts) == 0 {
explicitSteps := profile.Steps != nil
if len(profile.Artifacts) > 0 && explicitSteps {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must not declare both artifacts and steps", pipelineID)
}
if explicitSteps && len(options.Only) > 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q --only is not supported for explicit ordered steps", pipelineID)
}
steps := profile.Steps
if !explicitSteps {
steps = []PipelineStepProfile{{ID: "default", Artifacts: profile.Artifacts}}
}
if explicitSteps && len(steps) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must declare at least one ordered step", pipelineID)
}
if len(steps) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must declare at least one artifact lane", pipelineID)
}
@@ -194,15 +277,25 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
capabilities.add(chunkSpec.Provides...)
lanesByID, selectedLaneIDs, err := selectedArtifactLanes(pipelineID, profile.Artifacts, options)
if err != nil {
return ResolvedPipeline{}, err
if !explicitSteps {
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, profile.Artifacts, catalog); err != nil {
return ResolvedPipeline{}, err
}
} else {
allLanes := make(map[string]ArtifactLaneProfile)
for _, step := range profile.Steps {
for laneID, lane := range step.Artifacts {
allLanes[strings.TrimSpace(laneID)] = lane
}
}
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, allLanes, catalog); err != nil {
return ResolvedPipeline{}, err
}
}
if len(selectedLaneIDs) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
}
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
return ResolvedPipeline{}, err
for _, source := range profile.References {
if source.Artifact != nil {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q pipeline-level references may not use generated artifact selectors", pipelineID)
}
}
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
@@ -230,16 +323,55 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
outputCapabilities := capabilities.clone()
seenResolvedLanes := make(map[string]string)
seenResolvedSteps := make(map[string]struct{}, len(steps))
for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID]
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
for _, step := range steps {
stepID := strings.TrimSpace(step.ID)
if stepID == "" {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q step id must not be empty", pipelineID)
}
if _, exists := seenResolvedSteps[stepID]; exists {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q step id %q is duplicated after trimming", pipelineID, stepID)
}
seenResolvedSteps[stepID] = struct{}{}
laneOptions := ResolveOptions{}
if !explicitSteps {
laneOptions = options
}
lanesByID, selectedLaneIDs, err := selectedArtifactLanes(pipelineID, step.Artifacts, laneOptions)
if err != nil {
return ResolvedPipeline{}, err
}
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
outputCapabilities.addSet(laneCapabilities)
if len(selectedLaneIDs) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q step %q must declare at least one artifact lane", pipelineID, stepID)
}
if referenceSourceMapsConflict(profile.References, step.References) {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q step %q has a generated and external reference collision", pipelineID, stepID)
}
if err := validatePipelineReferenceDefaults(pipelineID, step.References, chunkSpec, lanesByID, catalog); err != nil {
return ResolvedPipeline{}, err
}
stepReferences := mergeReferenceMaps(profile.References, step.References)
resolvedStep := ResolvedPipelineStep{ID: stepID}
for _, laneID := range selectedLaneIDs {
if previousStep, exists := seenResolvedLanes[laneID]; exists {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q artifact lane id %q is duplicated across steps %q and %q", pipelineID, laneID, previousStep, stepID)
}
seenResolvedLanes[laneID] = stepID
laneProfile := lanesByID[laneID]
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, stepID, laneID, laneProfile, stepReferences, options, capabilities, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
resolvedStep.ArtifactLanes = append(resolvedStep.ArtifactLanes, lane)
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
outputCapabilities.addSet(laneCapabilities)
}
resolved.Steps = append(resolved.Steps, resolvedStep)
}
if err := validateGeneratedBindings(pipelineID, resolved, catalog); err != nil {
return ResolvedPipeline{}, err
}
outputSpec, err := outputSpec(catalog, resolved.Output.Module)
@@ -263,14 +395,16 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
func resolveArtifactLane(
pipelineID string,
stepID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
pipelineReferences map[string]ReferenceSource,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
lane := ResolvedArtifactLane{
StepID: strings.TrimSpace(stepID),
ID: laneID,
Extract: resolveBinding(profile.Extract, ""),
Merge: resolveBinding(profile.Merge, DefaultMergeModule),
@@ -294,6 +428,9 @@ func resolveArtifactLane(
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
if referenceSourceMapsConflict(profile.References, lane.Extract.References) {
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q step %q lane %q extract has a generated and external reference collision", pipelineID, stepID, laneID)
}
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
@@ -309,6 +446,7 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, err
}
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
@@ -332,6 +470,7 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, err
}
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
@@ -355,6 +494,7 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, err
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
capabilities.add(normalizeSpec.Provides...)
if len(lane.Validators) > 0 {
@@ -382,6 +522,83 @@ func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
}
func validateGeneratedBindings(pipelineID string, resolved ResolvedPipeline, catalog ModuleCatalog) error {
stepIndex := make(map[string]int, len(resolved.Steps))
laneIndex := make(map[string]ResolvedArtifactLane)
for index, step := range resolved.Steps {
if _, exists := stepIndex[step.ID]; exists {
return fmt.Errorf("pipeline %q step id %q is ambiguous", pipelineID, step.ID)
}
stepIndex[step.ID] = index
for _, lane := range step.ArtifactLanes {
laneIndex[step.ID+"\x00"+lane.ID] = lane
}
}
for consumerIndex, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
for _, target := range []ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
slots, err := resolvedReferenceSlots(target, lane.ArtifactKind, catalog)
if err != nil {
return fmt.Errorf("pipeline %q step %q lane %q: %w", pipelineID, step.ID, lane.ID, err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
slotByName[slot.Name] = slot
}
for _, binding := range target.Bindings {
if binding.Artifact == nil {
continue
}
producerStep, ok := stepIndex[strings.TrimSpace(binding.Artifact.Step)]
if !ok {
return fmt.Errorf("pipeline %q step %q lane %q %s reference slot %q producer step %q is not declared", pipelineID, step.ID, lane.ID, target.Stage, binding.SlotName, binding.Artifact.Step)
}
if producerStep >= consumerIndex {
return fmt.Errorf("pipeline %q step %q lane %q reference slot %q producer %q must be an earlier step", pipelineID, step.ID, lane.ID, binding.SlotName, binding.Artifact.Step)
}
producer, ok := laneIndex[strings.TrimSpace(binding.Artifact.Step)+"\x00"+strings.TrimSpace(binding.Artifact.Lane)]
if !ok {
return fmt.Errorf("pipeline %q step %q lane %q reference slot %q producer lane %q is not selected", pipelineID, step.ID, lane.ID, binding.SlotName, binding.Artifact.Lane)
}
slot, ok := slotByName[strings.TrimSpace(binding.SlotName)]
if !ok {
return fmt.Errorf("pipeline %q step %q lane %q %s reference slot %q is not declared", pipelineID, step.ID, lane.ID, target.Stage, binding.SlotName)
}
if len(slot.AcceptedArtifactKinds) == 0 {
return fmt.Errorf("pipeline %q step %q lane %q reference slot %q does not accept generated artifacts", pipelineID, step.ID, lane.ID, binding.SlotName)
}
accepted := false
for _, kind := range slot.AcceptedArtifactKinds {
if kind == producer.ArtifactKind {
accepted = true
break
}
}
if !accepted {
return fmt.Errorf("pipeline %q step %q lane %q reference slot %q does not accept artifact kind %q", pipelineID, step.ID, lane.ID, binding.SlotName, producer.ArtifactKind)
}
codecSpec, ok := catalog.ArtifactCodecs.Spec(producer.ArtifactKind)
if !ok {
return fmt.Errorf("pipeline %q step %q lane %q producer %q artifact codec %q is not registered", pipelineID, step.ID, lane.ID, producer.ID, producer.ArtifactKind)
}
if !referenceMediaTypeAccepted(codecSpec.MediaType, slot.AcceptedMediaTypes) {
return fmt.Errorf("pipeline %q step %q lane %q reference slot %q does not accept producer media type %q", pipelineID, step.ID, lane.ID, binding.SlotName, codecSpec.MediaType)
}
}
}
}
}
return nil
}
func resolvedReferenceSlots(target ResolvedReferenceTarget, artifactKind contracts.ArtifactKind, catalog ModuleCatalog) ([]contracts.ReferenceSlot, error) {
spec, err := referenceTargetSpec(target, artifactKind, catalog)
if err != nil {
return nil, err
}
return contracts.CloneReferenceSlots(spec.ReferenceSlots), nil
}
func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) {
if extractSpec.ArtifactKind == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", pipelineID, laneID, lane.Extract.Module)
@@ -593,23 +810,35 @@ func referenceTarget(stage ModuleStage, laneID string, module string, bindings [
}
}
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
func mergeReferenceMaps(base map[string]ReferenceSource, override map[string]ReferenceSource) map[string]ReferenceSource {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
out := make(map[string]ReferenceSource, len(base)+len(override))
for key, value := range base {
out[key] = value
out[key] = cloneReferenceSource(value)
}
for key, value := range override {
out[key] = value
out[key] = cloneReferenceSource(value)
}
return out
}
func referenceSourceMapsConflict(base, override map[string]ReferenceSource) bool {
for rawKey, left := range base {
key := strings.TrimSpace(rawKey)
for rawOverrideKey, right := range override {
if key == strings.TrimSpace(rawOverrideKey) && left.IsGenerated() != right.IsGenerated() {
return true
}
}
}
return false
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
pipelineReferences map[string]ReferenceSource,
chunkSpec ModuleSpec,
lanesByID map[string]ArtifactLaneProfile,
catalog ModuleCatalog,
@@ -680,8 +909,8 @@ type referenceResolutionTarget struct {
Stage ModuleStage
Module string
Slots []contracts.ReferenceSlot
PipelineReferences map[string]string
LocalReferences map[string]string
PipelineReferences map[string]ReferenceSource
LocalReferences map[string]ReferenceSource
Options ResolveOptions
}
@@ -692,24 +921,46 @@ func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]Referen
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
addBinding := func(slotName string, source ReferenceSource, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
}
if source == "" {
if source.Artifact == nil {
source.Path = strings.TrimSpace(source.Path)
}
if source.Artifact == nil && source.Path == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
}
if source.Artifact != nil {
if strings.TrimSpace(source.Path) != "" {
return fmt.Errorf("%s reference slot %q must contain exactly one source form", referenceTargetErrorContext(target), slotName)
}
artifact := *source.Artifact
artifact.Step = strings.TrimSpace(artifact.Step)
artifact.Lane = strings.TrimSpace(artifact.Lane)
if artifact.Step == "" || artifact.Lane == "" {
return fmt.Errorf("%s reference slot %q artifact selector step and lane must not be empty", referenceTargetErrorContext(target), slotName)
}
source.Artifact = &artifact
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
}
bindings[slotName] = ReferenceBinding{
if previous, exists := bindings[slotName]; exists && (previous.Artifact != nil || source.Artifact != nil) {
return fmt.Errorf("%s reference slot %q has conflicting generated and external bindings", referenceTargetErrorContext(target), slotName)
}
binding := ReferenceBinding{
LaneID: target.LaneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
if source.Artifact != nil {
binding.Artifact = source.Artifact
} else {
binding.Source = source.Path
}
bindings[slotName] = binding
return nil
}
@@ -748,7 +999,7 @@ func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]Referen
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
if err := addBinding(override.SlotName, ExternalReference(override.Source), strings.TrimSpace(source)); err != nil {
return nil, err
}
}
@@ -768,7 +1019,9 @@ func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]Referen
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
}
delete(bindings, slotName)
if binding, ok := bindings[slotName]; ok && binding.Artifact == nil {
delete(bindings, slotName)
}
}
for _, slot := range target.Slots {
@@ -843,11 +1096,11 @@ func referenceTargetSlotLabel(target referenceResolutionTarget) string {
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
func normalizedReferenceMap(values map[string]ReferenceSource, keyName string) (map[string]ReferenceSource, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
out := make(map[string]ReferenceSource, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
@@ -856,8 +1109,8 @@ func normalizedReferenceMap(values map[string]string, keyName string) (map[strin
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
source := cloneReferenceSource(rawSource)
if source.Artifact == nil && strings.TrimSpace(source.Path) == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
@@ -877,7 +1130,7 @@ func sortedArtifactLaneProfileKeys(values map[string]ArtifactLaneProfile) []stri
return keys
}
func sortedStringMapKeys(values map[string]string) []string {
func sortedStringMapKeys(values map[string]ReferenceSource) []string {
if len(values) == 0 {
return nil
}
@@ -942,11 +1195,11 @@ func cloneOptions(options map[string]any) map[string]any {
return copied
}
func normalizeReferenceMap(values map[string]string) map[string]string {
func normalizeReferenceMap(values map[string]ReferenceSource) map[string]ReferenceSource {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
out := make(map[string]ReferenceSource, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
@@ -956,7 +1209,7 @@ func normalizeReferenceMap(values map[string]string) map[string]string {
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
out[key] = cloneReferenceSource(values[rawByNormalized[key]])
}
return out
}
@@ -1009,7 +1262,7 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain
Output ModuleBinding
}{
@@ -1017,7 +1270,7 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
Input: resolved.Input,
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
Steps: resolved.Steps,
ValidatorChains: resolved.ValidatorChains,
Output: resolved.Output,
}