eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve extraction reference bindings from config

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 14:21:36 +00:00
parent 1c31f56af1
commit 70d733edaf
11 changed files with 625 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
internal/core/config/file_config.go
View File

@@ -35,10 +35,11 @@ type FileLLMProfile struct {
}
type FilePipelineProfile struct {
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileArtifactLaneProfile struct {
@@ -46,6 +47,7 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileConcurrencyConfig struct {
@@ -212,6 +214,18 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
}
}
for _, profileID := range profileIDs {
@@ -253,9 +267,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return err
}
profile := pipeline.PipelineProfile{
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -266,7 +281,8 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(),
Extract: fileLane.Extract.toPipelineBinding(),
References: normalizedStringMap(fileLane.References),
}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -318,6 +334,25 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil
}
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {