Resolve extraction reference bindings from config

This commit is contained in:
2026-07-05 14:21:36 +00:00
parent 1c31f56af1
commit 70d733edaf
11 changed files with 625 additions and 34 deletions

View File

@@ -30,6 +30,14 @@ before execution:
The CLI writes the resolved pipeline and digest to diagnostics. The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by
extractor reference slot name. During resolution, pipeline-level bindings act as
defaults for selected lanes whose extractor declares the slot, lane-level
bindings override or add lane bindings, and runtime unbinds can remove bindings.
Resolution validates bindings against extractor specs and records lane-scoped
binding metadata. It does not read reference files or include reference bytes in
source digests.
## Registries And Module Specs ## Registries And Module Specs
`pipeline.Registries` holds concrete constructors for execution. A `pipeline.Registries` holds concrete constructors for execution. A

View File

@@ -73,6 +73,7 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
out.References = cloneStringMap(in.References)
if len(in.Artifacts) > 0 { if len(in.Artifacts) > 0 {
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts)) out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
for key, lane := range in.Artifacts { for key, lane := range in.Artifacts {
@@ -87,6 +88,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = cloneStringMap(in.References)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {
@@ -96,6 +98,17 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
return out return out
} }
func cloneStringMap(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
if len(in.Options) > 0 { if len(in.Options) > 0 {

View File

@@ -14,12 +14,16 @@ type ResolveInput struct {
Only []string Only []string
Catalog pipeline.ModuleCatalog Catalog pipeline.ModuleCatalog
LLMProfileOverride string LLMProfileOverride string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
} }
type EffectiveConfig struct { type EffectiveConfig struct {
Config Config Config Config
PipelineID string PipelineID string
Only []string Only []string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
ResolvedPipeline pipeline.ResolvedPipeline ResolvedPipeline pipeline.ResolvedPipeline
} }
@@ -46,7 +50,11 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
applyLLMProfileOverride(&profile, override) applyLLMProfileOverride(&profile, override)
} }
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog) resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
if err != nil { if err != nil {
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err) return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
} }
@@ -55,6 +63,8 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
Config: cloneConfig(c), Config: cloneConfig(c),
PipelineID: pipelineID, PipelineID: pipelineID,
Only: append([]string(nil), input.Only...), Only: append([]string(nil), input.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
ResolvedPipeline: resolved, ResolvedPipeline: resolved,
}, nil }, nil
} }

View File

@@ -39,6 +39,7 @@ type FilePipelineProfile struct {
Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"` Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileArtifactLaneProfile struct { type FileArtifactLaneProfile struct {
@@ -46,6 +47,7 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"` Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"` Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"` Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileConcurrencyConfig struct { type FileConcurrencyConfig struct {
@@ -212,6 +214,18 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil { if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err return err
} }
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
}
} }
for _, profileID := range profileIDs { for _, profileID := range profileIDs {
@@ -256,6 +270,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID, ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(), Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
} }
if filePipeline.Chunk != nil { if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding() profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -267,6 +282,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
lane := pipeline.ArtifactLaneProfile{ lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(), Extract: fileLane.Extract.toPipelineBinding(),
References: normalizedStringMap(fileLane.References),
} }
if fileLane.Merge != nil { if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding() lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -318,6 +334,25 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil return keys, rawByNormalized, nil
} }
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName) name := strings.TrimSpace(envName)
if name == "" { if name == "" {

View File

@@ -144,6 +144,31 @@ pipelines:
} }
} }
func TestParseFileConfigReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
references:
" roster ": " ./shared-roster.yml "
artifacts:
events:
extract: fake/extract
references:
" lore ": " ./lore.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
}
gotLaneRefs := profile.Artifacts["events"].References
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1
@@ -297,6 +322,58 @@ pipelines:
} }
} }
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
tests := []struct {
name string
raw string
want string
}{
{
name: "pipeline",
raw: `
version: 1
pipelines:
example:
input: fake/input
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" reference slot`,
},
{
name: "lane",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" reference slot`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate reference slot error, got %v", err)
}
})
}
}
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) { func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1

View File

@@ -24,6 +24,8 @@ func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
Config: e.Config.Redacted(), Config: e.Config.Redacted(),
PipelineID: e.PipelineID, PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...), Only: append([]string(nil), e.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline), ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
} }
} }
@@ -47,6 +49,7 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = append([]pipeline.ReferenceBinding(nil), in.References...)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {

View File

@@ -3,6 +3,7 @@ package config
import ( import (
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -65,12 +66,21 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
lane := cfg.Pipelines["example"].Artifacts["events"] lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2} lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"].Artifacts["events"] = lane cfg.Pipelines["example"].Artifacts["events"] = lane
effective, err := cfg.Resolve(ResolveInput{ effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example", PipelineID: "example",
Only: []string{"events"}, Only: []string{"events"},
Catalog: fakeCatalog(t), Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
}) })
if err != nil { if err != nil {
t.Fatalf("Resolve: %v", err) t.Fatalf("Resolve: %v", err)
@@ -98,4 +108,8 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied") t.Fatalf("expected resolved pipeline options to be copied")
} }
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
} }

View File

@@ -98,11 +98,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil { if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
return err return err
} }
if err := validateReferenceMap(id, "", profile.References); err != nil {
return err
}
for rawLaneID, lane := range profile.Artifacts { for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID) laneID := strings.TrimSpace(rawLaneID)
if laneID == "" { if laneID == "" {
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id) return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
} }
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil { if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
return err return err
} }
@@ -122,6 +128,33 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil return nil
} }
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
}
if _, ok := seen[slotName]; ok {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
}
}
return nil
}
func validateBindingLLMProfile( func validateBindingLLMProfile(
pipelineID string, pipelineID string,
laneID string, laneID string,

View File

@@ -116,6 +116,73 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
} }
} }
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "reference slot", "empty"},
},
{
name: "empty pipeline source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "roster", "source", "empty"},
},
{
name: "empty lane slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "reference slot", "empty"},
},
{
name: "empty lane source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "roster", "source", "empty"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) { func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
const ( const (
@@ -28,6 +30,7 @@ type ArtifactLaneProfile struct {
Merge ModuleBinding `json:"merge,omitempty"` Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"` Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"` Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type PipelineProfile struct { type PipelineProfile struct {
@@ -36,10 +39,25 @@ type PipelineProfile struct {
Chunk ModuleBinding `json:"chunk,omitempty"` Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"` Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"` Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type ResolveOptions struct { type ResolveOptions struct {
Only []string Only []string
ReferenceOverrides []ReferenceBinding
ReferenceUnbinds []ReferenceUnbind
}
type ReferenceBinding struct {
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
LaneID string `json:"lane_id"`
SlotName string `json:"slot_name"`
} }
type ResolvedArtifactLane struct { type ResolvedArtifactLane struct {
@@ -48,6 +66,7 @@ type ResolvedArtifactLane struct {
Merge ModuleBinding Merge ModuleBinding
Normalize ModuleBinding Normalize ModuleBinding
Validators []ModuleBinding Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
} }
type ResolvedPipeline struct { type ResolvedPipeline struct {
@@ -126,7 +145,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
for _, laneID := range selectedLaneIDs { for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID] laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog) lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil { if err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -150,7 +169,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return resolved, nil return resolved, nil
} }
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) { func resolveArtifactLane(
pipelineID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
lane := ResolvedArtifactLane{ lane := ResolvedArtifactLane{
ID: laneID, ID: laneID,
Extract: resolveBinding(profile.Extract, ""), Extract: resolveBinding(profile.Extract, ""),
@@ -171,6 +198,11 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(extractSpec.Requires); ok { if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
} }
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -205,6 +237,162 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
return lane, capabilities, nil return lane, capabilities, nil
} }
func resolveReferenceBindings(
pipelineID string,
laneID string,
extractorModule string,
slots []contracts.ReferenceSlot,
pipelineReferences map[string]string,
laneReferences map[string]string,
options ResolveOptions,
) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
slotByName[slot.Name] = slot
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
if source == "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, laneID, slotName, extractorModule)
}
bindings[slotName] = ReferenceBinding{
LaneID: laneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := slotByName[slotName]; !ok {
continue
}
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
normalizedLaneReferences, err := normalizedReferenceMap(laneReferences, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLaneReferences) {
if err := addBinding(slotName, normalizedLaneReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range options.ReferenceOverrides {
optionLaneID := strings.TrimSpace(override.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference override lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
source := override.BindingSource
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
return nil, err
}
}
for _, unbind := range options.ReferenceUnbinds {
optionLaneID := strings.TrimSpace(unbind.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference unbind lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("pipeline %q lane %q reference unbind slot name must not be empty", pipelineID, laneID)
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared", pipelineID, laneID, slotName)
}
delete(bindings, slotName)
}
for _, slot := range slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q required reference slot %q is not bound", pipelineID, laneID, slot.Name)
}
}
}
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved, nil
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
}
return out, nil
}
func sortedStringMapKeys(values map[string]string) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding { func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module) module := strings.TrimSpace(binding.Module)
if module == "" { if module == "" {

View File

@@ -3,6 +3,7 @@ package pipeline
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@@ -125,6 +126,137 @@ func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
} }
} }
func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{
" roster ": " ./shared-roster.yml ",
"unclaimed": "./ignored.yml",
}
lane := profile.Artifacts["events"]
lane.References = map[string]string{
"roster": "./lane-roster.yml",
" lore ": " ./lore.md ",
}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
{Name: "lore"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events", "summaries"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.References, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References)
}
}
func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{"missing": "./missing.yml"}
profile.Artifacts["events"] = lane
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "missing", "not declared")
}
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
if _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"notes"}}, catalog); err != nil {
t.Fatalf("ResolvePipeline(unselected required slot) error = %v, want nil", err)
}
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"events"}}, catalog)
if err == nil {
t.Fatal("ResolvePipeline(selected required slot) error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineReferenceUnbindCanLeaveRequiredSlotMissing(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
_, err := ResolvePipeline(profile, ResolveOptions{
ReferenceUnbinds: []ReferenceUnbind{{LaneID: "events", SlotName: "roster"}},
}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog()
for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.Extractor, error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}
func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) { func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t)) _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t))
if err == nil { if err == nil {
@@ -463,6 +595,17 @@ func laneIDs(lanes []ResolvedArtifactLane) []string {
return ids return ids
} }
func resolvedLane(t *testing.T, lanes []ResolvedArtifactLane, laneID string) ResolvedArtifactLane {
t.Helper()
for _, lane := range lanes {
if lane.ID == laneID {
return lane
}
}
t.Fatalf("lane %q not found in %#v", laneID, laneIDs(lanes))
return ResolvedArtifactLane{}
}
func assertErrorContains(t *testing.T, err error, values ...string) { func assertErrorContains(t *testing.T, err error, values ...string) {
t.Helper() t.Helper()