eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reject ambiguous configuration and reference bindings

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-09 00:29:04 +00:00
parent 90c7fa6381
commit 41a8a80dda
7 changed files with 212 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
internal/framework/pipeline/profile.go
View File

@@ -305,7 +305,10 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return ResolvedPipeline{}, err
}
input := resolveBinding(profile.Input, "")
input, err := resolveBinding(profile.Input, "", fmt.Sprintf("pipeline %q input reference slot", pipelineID))
if err != nil {
return ResolvedPipeline{}, err
}
if input.Module == "" {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q input module must not be empty", pipelineID)
}
@@ -320,7 +323,10 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
capabilities.add(inputModuleSpec.Provides...)
chunk := resolveBinding(profile.Chunk, DefaultChunkModule)
chunk, err := resolveBinding(profile.Chunk, DefaultChunkModule, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID))
if err != nil {
return ResolvedPipeline{}, err
}
chunkSpec, err := chunkerSpec(catalog, chunk.Module)
if err != nil {
return ResolvedPipeline{}, moduleLookupError(pipelineID, "", StageChunk, chunk.Module, err)
@@ -363,6 +369,10 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if err != nil {
return ResolvedPipeline{}, err
}
output, err := resolveBinding(profile.Output, DefaultOutputModule, fmt.Sprintf("pipeline %q output reference slot", pipelineID))
if err != nil {
return ResolvedPipeline{}, err
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
@@ -370,7 +380,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
Chunk: chunk,
ChunkExecutionClass: chunkSpec.ExecutionClass,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
Output: output,
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
if err != nil {
@@ -485,13 +495,29 @@ func resolveArtifactLane(
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
extract, err := resolveBinding(profile.Extract, "", fmt.Sprintf("pipeline %q step %q lane %q extract reference slot", pipelineID, stepID, laneID))
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
merge, err := resolveBinding(profile.Merge, DefaultMergeModule, fmt.Sprintf("pipeline %q step %q lane %q merge reference slot", pipelineID, stepID, laneID))
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalize, err := resolveBinding(profile.Normalize, DefaultNormalizeModule, fmt.Sprintf("pipeline %q step %q lane %q normalize reference slot", pipelineID, stepID, laneID))
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
validators, err := resolveBindings(profile.Validators, "", fmt.Sprintf("pipeline %q step %q lane %q validator reference slot", pipelineID, stepID, laneID))
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
lane := ResolvedArtifactLane{
StepID: strings.TrimSpace(stepID),
ID: laneID,
Extract: resolveBinding(profile.Extract, ""),
Merge: resolveBinding(profile.Merge, DefaultMergeModule),
Normalize: resolveBinding(profile.Normalize, DefaultNormalizeModule),
Validators: resolveBindings(profile.Validators, ""),
Extract: extract,
Merge: merge,
Normalize: normalize,
Validators: validators,
}
if lane.Extract.Module == "" {
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
@@ -516,6 +542,7 @@ func resolveArtifactLane(
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
StepID: strings.TrimSpace(stepID),
LaneID: laneID,
Stage: StageExtract,
Module: lane.Extract.Module,
@@ -541,6 +568,7 @@ func resolveArtifactLane(
}
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
StepID: strings.TrimSpace(stepID),
LaneID: laneID,
Stage: StageMerge,
Module: lane.Merge.Module,
@@ -566,6 +594,7 @@ func resolveArtifactLane(
}
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
StepID: strings.TrimSpace(stepID),
LaneID: laneID,
Stage: StageNormalize,
Module: lane.Normalize.Module,
@@ -925,7 +954,7 @@ func validatePipelineReferenceDefaults(
lanesByID map[string]ArtifactLaneProfile,
catalog ModuleCatalog,
) error {
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
normalizedPipelineReferences, err := normalizeReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return err
}
@@ -939,38 +968,38 @@ func validatePipelineReferenceDefaults(
}
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
laneProfile := lanesByID[laneID]
extract := resolveBinding(laneProfile.Extract, "")
if extract.Module == "" {
extractModule := resolveModuleKey(laneProfile.Extract.Module, "")
if extractModule == "" {
return fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
}
extractSpec, err := extractorSpec(catalog, extract.Module)
extractSpec, err := extractorSpec(catalog, extractModule)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
return moduleLookupError(pipelineID, laneID, StageExtract, extractModule, err)
}
for _, slot := range extractSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
mergeModule := resolveModuleKey(laneProfile.Merge.Module, DefaultMergeModule)
var artifactType reflect.Type
artifactKind := extractSpec.ArtifactKind
if artifactKind != "" && catalog.Extractors != nil {
if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok {
if entry, ok := catalog.Extractors.typedEntry(extractModule); ok {
artifactType = entry.valueType
}
}
mergeSpec, err := mergerSpecForArtifact(catalog, merge.Module, artifactKind, artifactType)
mergeSpec, err := mergerSpecForArtifact(catalog, mergeModule, artifactKind, artifactType)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageMerge, merge.Module, err)
return moduleLookupError(pipelineID, laneID, StageMerge, mergeModule, err)
}
for _, slot := range mergeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpecForArtifact(catalog, normalize.Module, artifactKind, artifactType)
normalizeModule := resolveModuleKey(laneProfile.Normalize.Module, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpecForArtifact(catalog, normalizeModule, artifactKind, artifactType)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
return moduleLookupError(pipelineID, laneID, StageNormalize, normalizeModule, err)
}
for _, slot := range normalizeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
@@ -987,6 +1016,7 @@ func validatePipelineReferenceDefaults(
type referenceResolutionTarget struct {
PipelineID string
StepID string
LaneID string
Stage ModuleStage
Module string
@@ -1046,7 +1076,7 @@ func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]Referen
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
normalizedPipelineReferences, err := normalizeReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
if err != nil {
return nil, err
}
@@ -1059,7 +1089,7 @@ func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]Referen
}
}
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
normalizedLocalReferences, err := normalizeReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
if err != nil {
return nil, err
}
@@ -1166,6 +1196,9 @@ func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBi
func referenceTargetErrorContext(target referenceResolutionTarget) string {
if target.LaneID != "" {
if target.StepID != "" {
return fmt.Sprintf("pipeline %q step %q lane %q %s module %q", target.PipelineID, target.StepID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
@@ -1173,17 +1206,22 @@ func referenceTargetErrorContext(target referenceResolutionTarget) string {
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
if target.LaneID != "" {
if target.StepID != "" {
return fmt.Sprintf("pipeline %q step %q lane %q %s reference slot", target.PipelineID, target.StepID, target.LaneID, target.Stage)
}
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
}
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}
func normalizedReferenceMap(values map[string]ReferenceSource, keyName string) (map[string]ReferenceSource, error) {
func normalizeReferenceMap(values map[string]ReferenceSource, keyName string) (map[string]ReferenceSource, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]ReferenceSource, len(values))
for rawSlotName, rawSource := range values {
rawSlotNames := sortedStringMapKeys(values)
for _, rawSlotName := range rawSlotNames {
rawSource := values[rawSlotName]
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
@@ -1236,20 +1274,29 @@ func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module)
if module == "" {
module = defaultModule
}
func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLabel string) (ModuleBinding, error) {
module := resolveModuleKey(binding.Module, defaultModule)
llmProfile := strings.TrimSpace(binding.LLMProfile)
references, err := normalizeReferenceMap(binding.References, referenceSlotLabel)
if err != nil {
return ModuleBinding{}, err
}
return ModuleBinding{
Module: module,
LLMProfile: llmProfile,
Retries: binding.Retries,
Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
References: references,
Validators: cloneValidatorOverride(binding.Validators),
}, nil
}
func resolveModuleKey(module string, defaultModule string) string {
module = strings.TrimSpace(module)
if module == "" {
return defaultModule
}
return module
}
func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, overrideProfile string) error {
@@ -1312,17 +1359,20 @@ func applyEffectiveLLMProfiles(resolved *ResolvedPipeline, pipelineProfile, over
return nil
}
func resolveBindings(bindings []ModuleBinding, defaultModule string) []ModuleBinding {
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
if len(bindings) == 0 {
return nil
return nil, nil
}
resolved := make([]ModuleBinding, 0, len(bindings))
for _, binding := range bindings {
resolvedBinding := resolveBinding(binding, defaultModule)
for index, binding := range bindings {
resolvedBinding, err := resolveBinding(binding, defaultModule, fmt.Sprintf("%s %d", referenceSlotLabel, index))
if err != nil {
return nil, err
}
resolved = append(resolved, resolvedBinding)
}
return resolved
return resolved, nil
}
func cloneOptions(options map[string]any) map[string]any {
@@ -1358,25 +1408,6 @@ func cloneOptionValue(value any) any {
}
}
func normalizeReferenceMap(values map[string]ReferenceSource) map[string]ReferenceSource {
if len(values) == 0 {
return nil
}
out := make(map[string]ReferenceSource, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = cloneReferenceSource(values[rawByNormalized[key]])
}
return out
}
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
for rawLaneID, lane := range artifacts {