Add stage-local reference config bindings

This commit is contained in:
2026-07-05 16:18:00 +00:00
parent 9278797aa9
commit 51053d390d
8 changed files with 437 additions and 39 deletions

View File

@@ -135,8 +135,8 @@ Artifact lane fields:
- `normalize`: optional module binding. Default module is `noop`. - `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently - `validators`: optional list of module bindings. The production CLI currently
does not register validator modules. does not register validator modules.
- `references`: optional map of extractor reference slot names to reference - `references`: optional compatibility alias for extractor reference bindings.
paths. Lane bindings override pipeline-level bindings for the same slot. Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline `notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible against the production module catalog and fail fast for unknown or incompatible
@@ -144,15 +144,15 @@ module keys.
Reference bindings are validated against extractor-declared slots during Reference bindings are validated against extractor-declared slots during
pipeline resolution. Required slots must be bound after config defaults, pipeline resolution. Required slots must be bound after config defaults,
lane-level bindings, and run-time `--reference` or `--without-reference` extractor binding references, lane-level compatibility bindings, and run-time
overrides are applied. Config-relative paths are resolved relative to the `--reference` or `--without-reference` overrides are applied. Config-relative
config file; CLI reference paths are resolved relative to the current working paths are resolved relative to the config file; CLI reference paths are resolved
directory. Bound files must be UTF-8 text and are passed only to lane relative to the current working directory. Bound files must be UTF-8 text and
extractors that declare the slot. Reference media types are inferred from file are passed only to lane extractors that declare the slot. Reference media types
extensions, recorded as canonical base media types, and checked only when a are inferred from file extensions, recorded as canonical base media types, and
module declares `AcceptedMediaTypes`; unknown extensions are recorded as checked only when a module declares `AcceptedMediaTypes`; unknown extensions are
`application/octet-stream`. Reference content is not written to diagnostics, recorded as `application/octet-stream`. Reference content is not written to
logs, errors, or manifests. diagnostics, logs, errors, or manifests.
Pipeline-level `references` are defaults. They are valid when at least one Pipeline-level `references` are defaults. They are valid when at least one
declared lane in the pipeline has an extractor that declares the slot. During a declared lane in the pipeline has an extractor that declares the slot. During a
@@ -170,7 +170,9 @@ pipelines:
extract: dnd/spells extract: dnd/spells
``` ```
Lane-level `references` override or add bindings for one lane: Extractor binding `references` are the canonical lane-local location. The
legacy lane-level `references` field remains supported as an alias; when both
bind the same slot, `extract.references` wins:
```yaml ```yaml
pipelines: pipelines:
@@ -180,11 +182,19 @@ pipelines:
glossary: ./campaign/glossary.txt glossary: ./campaign/glossary.txt
artifacts: artifacts:
spells: spells:
extract: dnd/spells references:
roster: ./campaign/legacy-roster.txt
extract:
module: dnd/spells
references: references:
roster: ./campaign/session-roster.txt roster: ./campaign/session-roster.txt
``` ```
`chunk.references` and `normalize.references` are accepted in object-form
bindings and preserved in the effective configuration. They are validated as
reference maps, but current reference materialization still delivers content
only to extractor bindings.
## Module Bindings ## Module Bindings
Every module binding may use shorthand: Every module binding may use shorthand:
@@ -208,6 +218,9 @@ Binding fields:
- `module`: module key. - `module`: module key.
- `llm_profile`: optional LLM profile ID. Empty means `default`. - `llm_profile`: optional LLM profile ID. Empty means `default`.
- `options`: optional module-specific settings. - `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
`output` bindings reject this field during validation.
The `--llm-profile` run flag overrides every effective module binding to use The `--llm-profile` run flag overrides every effective module binding to use
one configured profile. one configured profile.

View File

@@ -114,6 +114,7 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
if len(in.Options) > 0 { if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options) out.Options = cloneOptions(in.Options)
} }
out.References = cloneStringMap(in.References)
return out return out
} }

View File

@@ -99,6 +99,7 @@ type fileModuleBinding struct {
Module string Module string
LLMProfile string LLMProfile string
Options map[string]any Options map[string]any
References map[string]string
} }
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err return err
} }
b.Options = normalizeOptions(options) b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
default: default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value) return fmt.Errorf("field %s not found in module binding", keyNode.Value)
} }
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module), Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile), LLMProfile: strings.TrimSpace(b.LLMProfile),
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
} }
} }
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil { if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err return err
} }
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts { for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID) laneID := strings.TrimSpace(rawLaneID)
if laneID == "" { if laneID == "" {
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil { if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err return err
} }
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
} }
} }
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
} }
for _, laneID := range laneIDs { for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{ lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(), Extract: extract,
References: normalizedStringMap(fileLane.References), References: normalizedStringMap(fileLane.References),
} }
if fileLane.Merge != nil { if fileLane.Merge != nil {
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
return out return out
} }
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName) name := strings.TrimSpace(envName)
if name == "" { if name == "" {

View File

@@ -169,6 +169,53 @@ pipelines:
} }
} }
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
" scene_guide ": " ./scenes.md "
artifacts:
events:
extract:
module: fake/extract
references:
" glossary ": " ./glossary.md "
" roster ": " ./extract-roster.yml "
references:
roster: ./legacy-roster.yml
lore: ./lore.md
normalize:
module: noop
references:
" normalization_notes ": " ./normalization.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
}
lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
}
wantExtract := map[string]string{
"glossary": "./glossary.md",
"lore": "./lore.md",
"roster": "./extract-roster.yml",
}
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1
@@ -357,6 +404,56 @@ pipelines:
`, `,
want: `pipeline "example" lane "events" reference slot`, want: `pipeline "example" lane "events" reference slot`,
}, },
{
name: "chunk",
raw: `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" chunk reference slot`,
},
{
name: "extract",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" extract reference slot`,
},
{
name: "normalize",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
normalize:
module: noop
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" normalize reference slot`,
},
} }
for _, tc := range tests { for _, tc := range tests {

View File

@@ -67,7 +67,12 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
lane := cfg.Pipelines["example"].Artifacts["events"] lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2} lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"} lane.References = map[string]string{"roster": "./roster.yml"}
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
cfg.Pipelines["example"].Artifacts["events"] = lane cfg.Pipelines["example"].Artifacts["events"] = lane
pipelineProfile := cfg.Pipelines["example"]
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
cfg.Pipelines["example"] = pipelineProfile
effective, err := cfg.Resolve(ResolveInput{ effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example", PipelineID: "example",
@@ -78,6 +83,7 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
Requires: []string{"chunks"}, Requires: []string{"chunks"},
Provides: []string{"artifact"}, Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{ ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary"},
{Name: "roster"}, {Name: "roster"},
}, },
}), }),
@@ -109,9 +115,21 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
t.Fatalf("expected resolved pipeline options to be copied") t.Fatalf("expected resolved pipeline options to be copied")
} }
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml" payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source != "./roster.yml" { if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied") t.Fatalf("expected resolved pipeline references to be copied")
} }
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
t.Fatalf("expected chunk references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
t.Fatalf("expected extract references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
t.Fatalf("expected normalize references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{ effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{ Slots: map[string]contracts.ResolvedReferenceSlot{
@@ -136,3 +154,12 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
t.Fatalf("expected materialized reference content to be copied, got %q", got) t.Fatalf("expected materialized reference content to be copied, got %q", got)
} }
} }
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
for _, binding := range bindings {
if binding.SlotName == slotName {
return binding.Source
}
}
return ""
}

View File

@@ -89,13 +89,13 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if profile.ID != "" && strings.TrimSpace(profile.ID) != id { if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID) return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
} }
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil { if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil { if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil { if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
return err return err
} }
if err := validateReferenceMap(id, "", profile.References); err != nil { if err := validateReferenceMap(id, "", profile.References); err != nil {
@@ -109,17 +109,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if err := validateReferenceMap(id, laneID, lane.References); err != nil { if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil { if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil { if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil { if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
return err return err
} }
for i, validator := range lane.Validators { for i, validator := range lane.Validators {
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil { if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
return err return err
} }
} }
@@ -128,33 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil return nil
} }
func validateBinding(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
profiles map[string]LLMProfile,
referencesAllowed bool,
) error {
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
if !referencesAllowed {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
}
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error { func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
seen := make(map[string]struct{}, len(references)) seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references { for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName) slotName := strings.TrimSpace(rawSlotName)
if slotName == "" { if slotName == "" {
if laneID != "" { return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
} }
if _, ok := seen[slotName]; ok { if _, ok := seen[slotName]; ok {
if laneID != "" { return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
} }
seen[slotName] = struct{}{} seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" { if strings.TrimSpace(rawSource) == "" {
if laneID != "" { return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
} }
} }
return nil return nil
} }
func referenceContext(pipelineID string, laneID string, slot string) string {
if laneID != "" && slot != "" {
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
}
if laneID != "" {
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
}
if slot != "" {
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
}
return fmt.Sprintf("pipeline %q", pipelineID)
}
func validateBindingLLMProfile( func validateBindingLLMProfile(
pipelineID string, pipelineID string,
laneID string, laneID string,

View File

@@ -122,6 +122,74 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
mutate func(Config) Config mutate func(Config) Config
want []string want []string
}{ }{
{
name: "empty chunk slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "reference slot", "empty"},
},
{
name: "empty chunk source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "roster", "source", "empty"},
},
{
name: "empty extract slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "reference slot", "empty"},
},
{
name: "empty extract source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "roster", "source", "empty"},
},
{
name: "empty normalize slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "reference slot", "empty"},
},
{
name: "empty normalize source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
},
{ {
name: "empty pipeline slot", name: "empty pipeline slot",
mutate: func(cfg Config) Config { mutate: func(cfg Config) Config {
@@ -183,6 +251,76 @@ func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
} }
} }
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "input",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Input.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "input", "references", "not supported"},
},
{
name: "merge",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "merge", "references", "not supported"},
},
{
name: "validator",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Validators = []pipeline.ModuleBinding{{
Module: "fake/validator",
References: map[string]string{"roster": "./roster.yml"},
}}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "validator[0]", "references", "not supported"},
},
{
name: "output",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Output.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "output", "references", "not supported"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) { func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -23,6 +23,7 @@ type ModuleBinding struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type ArtifactLaneProfile struct { type ArtifactLaneProfile struct {
@@ -212,7 +213,8 @@ func resolveArtifactLane(
if missing, ok := capabilities.missing(extractSpec.Requires); ok { if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
} }
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options) extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, extractReferences, options)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, err return ResolvedArtifactLane{}, nil, err
} }
@@ -261,6 +263,20 @@ func referenceTarget(stage ModuleStage, laneID string, module string, bindings [
} }
} }
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func validatePipelineReferenceDefaults( func validatePipelineReferenceDefaults(
pipelineID string, pipelineID string,
pipelineReferences map[string]string, pipelineReferences map[string]string,
@@ -480,6 +496,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Module: module, Module: module,
LLMProfile: llmProfile, LLMProfile: llmProfile,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
} }
} }
@@ -508,6 +525,25 @@ func cloneOptions(options map[string]any) map[string]any {
return copied return copied
} }
func normalizeReferenceMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) { func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts)) lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
for rawLaneID, lane := range artifacts { for rawLaneID, lane := range artifacts {