Add stage-local reference config bindings

This commit is contained in:
2026-07-05 16:18:00 +00:00
parent 9278797aa9
commit 51053d390d
8 changed files with 437 additions and 39 deletions

View File

@@ -99,6 +99,7 @@ type fileModuleBinding struct {
Module string
LLMProfile string
Options map[string]any
References map[string]string
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
}
}
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
}
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(),
Extract: extract,
References: normalizedStringMap(fileLane.References),
}
if fileLane.Merge != nil {
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
return out
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {