eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve extraction reference bindings from config

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 14:21:36 +00:00
parent 1c31f56af1
commit 70d733edaf
11 changed files with 625 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
internal/framework/pipeline/profile.go
View File

@@ -7,6 +7,8 @@ import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
@@ -24,22 +26,38 @@ type ModuleBinding struct {
}
type ArtifactLaneProfile struct {
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type PipelineProfile struct {
ID string `json:"id"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"`
ID string `json:"id"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type ResolveOptions struct {
Only []string
Only []string
ReferenceOverrides []ReferenceBinding
ReferenceUnbinds []ReferenceUnbind
}
type ReferenceBinding struct {
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
LaneID string `json:"lane_id"`
SlotName string `json:"slot_name"`
}
type ResolvedArtifactLane struct {
@@ -48,6 +66,7 @@ type ResolvedArtifactLane struct {
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
}
type ResolvedPipeline struct {
@@ -126,7 +145,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog)
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
@@ -150,7 +169,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return resolved, nil
}
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) {
func resolveArtifactLane(
pipelineID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
lane := ResolvedArtifactLane{
ID: laneID,
Extract: resolveBinding(profile.Extract, ""),
@@ -171,6 +198,11 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
}
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -205,6 +237,162 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
return lane, capabilities, nil
}
func resolveReferenceBindings(
pipelineID string,
laneID string,
extractorModule string,
slots []contracts.ReferenceSlot,
pipelineReferences map[string]string,
laneReferences map[string]string,
options ResolveOptions,
) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
slotByName[slot.Name] = slot
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
if source == "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, laneID, slotName, extractorModule)
}
bindings[slotName] = ReferenceBinding{
LaneID: laneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := slotByName[slotName]; !ok {
continue
}
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
normalizedLaneReferences, err := normalizedReferenceMap(laneReferences, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLaneReferences) {
if err := addBinding(slotName, normalizedLaneReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range options.ReferenceOverrides {
optionLaneID := strings.TrimSpace(override.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference override lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
source := override.BindingSource
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
return nil, err
}
}
for _, unbind := range options.ReferenceUnbinds {
optionLaneID := strings.TrimSpace(unbind.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference unbind lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("pipeline %q lane %q reference unbind slot name must not be empty", pipelineID, laneID)
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared", pipelineID, laneID, slotName)
}
delete(bindings, slotName)
}
for _, slot := range slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q required reference slot %q is not bound", pipelineID, laneID, slot.Name)
}
}
}
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved, nil
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
}
return out, nil
}
func sortedStringMapKeys(values map[string]string) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module)
if module == "" {