Add stage-local reference config bindings
This commit is contained in:
@@ -99,6 +99,7 @@ type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
Options map[string]any
|
||||
References map[string]string
|
||||
}
|
||||
|
||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
@@ -133,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
return err
|
||||
}
|
||||
b.Options = normalizeOptions(options)
|
||||
case "references":
|
||||
var references map[string]string
|
||||
if err := valueNode.Decode(&references); err != nil {
|
||||
return err
|
||||
}
|
||||
b.References = references
|
||||
default:
|
||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||
}
|
||||
@@ -148,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
Module: strings.TrimSpace(b.Module),
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
Options: cloneOptions(b.Options),
|
||||
References: normalizedStringMap(b.References),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,6 +225,19 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if filePipeline.Output != nil {
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
||||
laneID := strings.TrimSpace(rawLaneID)
|
||||
if laneID == "" {
|
||||
@@ -225,6 +246,24 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if fileLane.Merge != nil {
|
||||
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if fileLane.Normalize != nil {
|
||||
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i, validator := range fileLane.Validators {
|
||||
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,8 +319,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
for _, laneID := range laneIDs {
|
||||
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
||||
extract := fileLane.Extract.toPipelineBinding()
|
||||
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
||||
lane := pipeline.ArtifactLaneProfile{
|
||||
Extract: fileLane.Extract.toPipelineBinding(),
|
||||
Extract: extract,
|
||||
References: normalizedStringMap(fileLane.References),
|
||||
}
|
||||
if fileLane.Merge != nil {
|
||||
@@ -353,6 +394,20 @@ func normalizedStringMap(values map[string]string) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
||||
if len(base) == 0 && len(override) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(base)+len(override))
|
||||
for key, value := range base {
|
||||
out[key] = value
|
||||
}
|
||||
for key, value := range override {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
|
||||
Reference in New Issue
Block a user