eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement ordered pipeline step resolution

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-21 21:05:28 +00:00
parent f5618d1f0c
commit f846f252c0
44 changed files with 1335 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

337
internal/core/config/file_config.go
View File

@@ -28,19 +28,88 @@ type FileScriptoriumConfig struct {
}
type FilePipelineProfile struct {
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
}
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile")
if err != nil {
return err
}
*p = FilePipelineProfile(decoded)
_, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"]
return nil
}
func (s *FilePipelineStepProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineStepProfile FilePipelineStepProfile
var decoded plainFilePipelineStepProfile
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"id": {}, "artifacts": {}, "references": {},
}, "pipeline step"); err != nil {
return err
}
*s = FilePipelineStepProfile(decoded)
return nil
}
func (l *FileArtifactLaneProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFileArtifactLaneProfile FileArtifactLaneProfile
var decoded plainFileArtifactLaneProfile
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"extract": {}, "merge": {}, "normalize": {}, "validators": {}, "references": {},
}, "artifact lane"); err != nil {
return err
}
*l = FileArtifactLaneProfile(decoded)
return nil
}
func decodeKnownMapping(node *yaml.Node, target any, allowed map[string]struct{}, context string) (map[string]struct{}, error) {
if node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s must be an object", context)
}
if err := node.Decode(target); err != nil {
return nil, err
}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s field %q is duplicated", context, key)
}
if _, ok := allowed[key]; !ok {
return nil, fmt.Errorf("field %s not found in %s", key, context)
}
seen[key] = struct{}{}
}
return seen, nil
}
type FilePipelineStepProfile struct {
ID string `yaml:"id"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
}
type FileArtifactLaneProfile struct {
Extract fileModuleBinding `yaml:"extract"`
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
Extract fileModuleBinding `yaml:"extract"`
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
}
type FileConcurrencyConfig struct {
@@ -72,10 +141,90 @@ type fileModuleBinding struct {
LLMProfile string
Retries int
Options map[string]any
References map[string]string
References map[string]fileReferenceSource
Validators pipeline.ValidatorOverride
}
type fileReferenceSource struct {
path string
artifact *pipeline.ArtifactReference
}
func (source *fileReferenceSource) UnmarshalYAML(node *yaml.Node) error {
if source == nil {
return fmt.Errorf("reference source must not be nil")
}
switch node.Kind {
case yaml.ScalarNode:
if node.Tag != "!!str" {
return fmt.Errorf("external reference path must be a string")
}
path := strings.TrimSpace(node.Value)
if path == "" {
return fmt.Errorf("external reference path must not be empty")
}
source.path = path
source.artifact = nil
return nil
case yaml.MappingNode:
if len(node.Content) != 2 || node.Content[0].Value != "artifact" {
return fmt.Errorf("reference source mapping must contain only artifact")
}
artifactNode := node.Content[1]
if artifactNode.Kind != yaml.MappingNode {
return fmt.Errorf("artifact reference must be an object")
}
var step, lane string
seen := map[string]bool{}
for i := 0; i < len(artifactNode.Content); i += 2 {
key := artifactNode.Content[i].Value
value := artifactNode.Content[i+1]
if seen[key] {
return fmt.Errorf("artifact reference field %q is duplicated", key)
}
seen[key] = true
if value.Tag != "!!str" {
return fmt.Errorf("artifact reference field %q must be a string", key)
}
switch key {
case "step":
step = strings.TrimSpace(value.Value)
case "lane":
lane = strings.TrimSpace(value.Value)
default:
return fmt.Errorf("field %s not found in artifact reference", key)
}
}
if step == "" || lane == "" {
return fmt.Errorf("artifact reference step and lane must not be empty")
}
source.path = ""
source.artifact = &pipeline.ArtifactReference{Step: step, Lane: lane}
return nil
default:
return fmt.Errorf("reference source must be a string or object")
}
}
func (source fileReferenceSource) toPipelineSource() pipeline.ReferenceSource {
if source.artifact != nil {
artifact := *source.artifact
return pipeline.ReferenceSource{Artifact: &artifact}
}
return pipeline.ExternalReference(source.path)
}
func fileReferenceSourcesToPipeline(values map[string]fileReferenceSource) map[string]pipeline.ReferenceSource {
if len(values) == 0 {
return nil
}
out := make(map[string]pipeline.ReferenceSource, len(values))
for key, value := range values {
out[strings.TrimSpace(key)] = value.toPipelineSource()
}
return out
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
@@ -115,7 +264,7 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]string
var references map[string]fileReferenceSource
if err := valueNode.Decode(&references); err != nil {
return err
}
@@ -146,7 +295,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
LLMProfile: strings.TrimSpace(b.LLMProfile),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
References: fileReferenceSourcesToPipeline(b.References),
Validators: b.Validators,
}
}
@@ -214,10 +363,49 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
for _, pipelineID := range pipelineIDs {
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
hasArtifacts := filePipeline.artifactsSet || filePipeline.Artifacts != nil
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
if hasArtifacts && hasSteps {
return fmt.Errorf("pipeline %q must not declare both artifacts and steps", pipelineID)
}
if hasSteps && len(filePipeline.Steps) == 0 {
return fmt.Errorf("pipeline %q must declare at least one ordered step", pipelineID)
}
if hasSteps {
seenSteps := make(map[string]struct{}, len(filePipeline.Steps))
seenLanes := make(map[string]struct{})
for index, step := range filePipeline.Steps {
stepID := strings.TrimSpace(step.ID)
if stepID == "" {
return fmt.Errorf("pipeline %q step[%d] id must not be empty", pipelineID, index)
}
if _, ok := seenSteps[stepID]; ok {
return fmt.Errorf("pipeline %q step id %q is duplicated after trimming", pipelineID, stepID)
}
seenSteps[stepID] = struct{}{}
laneIDs, rawLaneIDs, err := normalizedMapKeys(step.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
if err != nil {
return err
}
for _, laneID := range laneIDs {
if _, ok := seenLanes[laneID]; ok {
return fmt.Errorf("pipeline %q artifact lane id %q is duplicated across steps", pipelineID, laneID)
}
seenLanes[laneID] = struct{}{}
fileLane := step.Artifacts[rawLaneIDs[laneID]]
if err := validateFileLaneReferences(pipelineID, stepID, laneID, fileLane); err != nil {
return err
}
}
if err := validateFileReferenceSources(step.References, fmt.Sprintf("pipeline %q step %q reference slot", pipelineID, stepID)); err != nil {
return err
}
}
}
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
if err := validateFileReferenceSources(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
@@ -281,6 +469,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
for _, pipelineID := range pipelineIDs {
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
if err != nil {
return err
@@ -289,7 +478,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
References: fileReferenceSourcesToPipeline(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -300,10 +489,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: extract,
References: normalizedStringMap(fileLane.References),
References: fileReferenceSourcesToPipeline(fileLane.References),
}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -319,6 +508,42 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
profile.Artifacts[laneID] = lane
}
if hasSteps {
profile.Artifacts = nil
profile.Steps = make([]pipeline.PipelineStepProfile, len(filePipeline.Steps))
for i, fileStep := range filePipeline.Steps {
stepID := strings.TrimSpace(fileStep.ID)
step := pipeline.PipelineStepProfile{
ID: stepID,
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(fileStep.Artifacts)),
References: fileReferenceSourcesToPipeline(fileStep.References),
}
stepLaneIDs, stepRawLaneIDs, err := normalizedMapKeys(fileStep.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
if err != nil {
return err
}
for _, laneID := range stepLaneIDs {
fileLane := fileStep.Artifacts[stepRawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{Extract: extract, References: fileReferenceSourcesToPipeline(fileLane.References)}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
}
if fileLane.Normalize != nil {
lane.Normalize = fileLane.Normalize.toPipelineBinding()
}
if len(fileLane.Validators) > 0 {
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
for index, validator := range fileLane.Validators {
lane.Validators[index] = validator.toPipelineBinding()
}
}
step.Artifacts[laneID] = lane
}
profile.Steps[i] = step
}
}
c.Pipelines[pipelineID] = profile
}
@@ -430,30 +655,72 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil
}
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
func validateFileReferenceSources(values map[string]fileReferenceSource, context string) error {
seen := make(map[string]struct{}, len(values))
for rawSlot, source := range values {
slot := strings.TrimSpace(rawSlot)
if slot == "" {
return fmt.Errorf("%s must not be empty", context)
}
if _, ok := seen[slot]; ok {
return fmt.Errorf("%s %q is duplicated after trimming", context, slot)
}
seen[slot] = struct{}{}
if source.artifact != nil {
if strings.TrimSpace(source.artifact.Step) == "" || strings.TrimSpace(source.artifact.Lane) == "" {
return fmt.Errorf("%s %q artifact selector step and lane must not be empty", context, slot)
}
if strings.TrimSpace(source.path) != "" {
return fmt.Errorf("%s %q must contain either an external path or artifact selector", context, slot)
}
continue
}
if strings.TrimSpace(source.path) == "" {
return fmt.Errorf("%s %q source must not be empty", context, slot)
}
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
return nil
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
func validateFileLaneReferences(pipelineID, stepID, laneID string, lane FileArtifactLaneProfile) error {
prefix := fmt.Sprintf("pipeline %q step %q lane %q", pipelineID, stepID, laneID)
references := []struct {
label string
values map[string]fileReferenceSource
}{
{label: "reference slot", values: lane.References},
{label: "extract reference slot", values: lane.Extract.References},
}
if lane.Merge != nil {
references = append(references, struct {
label string
values map[string]fileReferenceSource
}{label: "merge reference slot", values: lane.Merge.References})
}
if lane.Normalize != nil {
references = append(references, struct {
label string
values map[string]fileReferenceSource
}{label: "normalize reference slot", values: lane.Normalize.References})
}
for _, item := range references {
if err := validateFileReferenceSources(item.values, prefix+" "+item.label); err != nil {
return err
}
}
for index, validator := range lane.Validators {
if err := validateFileReferenceSources(validator.References, fmt.Sprintf("%s validator[%d] reference slot", prefix, index)); err != nil {
return err
}
}
return nil
}
func mergeReferenceSources(base map[string]pipeline.ReferenceSource, override map[string]pipeline.ReferenceSource) map[string]pipeline.ReferenceSource {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
out := make(map[string]pipeline.ReferenceSource, len(base)+len(override))
for key, value := range base {
out[key] = value
}