225 lines
7.4 KiB
Go
225 lines
7.4 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type pipelineResolutionMetadata struct {
|
|
rootPath string
|
|
imports []string
|
|
sources []string
|
|
ownership []pipelineFieldOwnership
|
|
}
|
|
|
|
type pipelineFieldOwnership struct {
|
|
path string
|
|
sources []string
|
|
}
|
|
|
|
type pipelineCompositionEnvelope struct {
|
|
imports []string
|
|
}
|
|
|
|
func loadComposedPipeline(path string) (*PipelineConfig, error) {
|
|
rootPath, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
|
|
}
|
|
rootFile, err := os.Open(rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
|
|
}
|
|
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
|
|
closeErr := rootFile.Close()
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
|
|
}
|
|
|
|
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imports, err := loadPipelineImports(rootPath, envelope.imports)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
documents := make([]*compositionDocument, 0, len(imports)+1)
|
|
documents = append(documents, baseRoot)
|
|
for _, imported := range imports {
|
|
documents = append(documents, imported.document)
|
|
}
|
|
merged, err := mergeAdditiveCompositions(documents...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rendered, err := merged.canonicalYAML()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var cfg PipelineConfig
|
|
if err := decodeStrictYAMLFromReader("pipeline", rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
|
|
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
|
|
}
|
|
records, err := merged.semanticRecords()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
metadata := &pipelineResolutionMetadata{
|
|
rootPath: rootPath,
|
|
sources: append([]string(nil), merged.sources...),
|
|
}
|
|
for _, imported := range imports {
|
|
metadata.imports = append(metadata.imports, imported.path)
|
|
}
|
|
for _, record := range records {
|
|
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
|
|
path: record.Path, sources: append([]string(nil), record.Sources...),
|
|
})
|
|
}
|
|
cfg.resolution = metadata
|
|
return &cfg, nil
|
|
}
|
|
|
|
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
|
|
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
|
|
return nil, pipelineCompositionEnvelope{}, err
|
|
}
|
|
root := cloneCompositionNode(document.root)
|
|
index := compositionFieldIndex(root.fields, "composition")
|
|
if index < 0 {
|
|
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, pipelineCompositionEnvelope{}, nil
|
|
}
|
|
envelopeNode := root.fields[index].value
|
|
if envelopeNode.kind != yaml.MappingNode {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition: expected a mapping, got %s",
|
|
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
|
|
)
|
|
}
|
|
|
|
var envelope pipelineCompositionEnvelope
|
|
for _, field := range envelopeNode.fields {
|
|
switch field.key {
|
|
case "imports":
|
|
if field.value.kind != yaml.SequenceNode {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.imports: expected a list, got %s",
|
|
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
|
|
)
|
|
}
|
|
for itemIndex, item := range field.value.items {
|
|
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.imports[%d]: expected a string path",
|
|
formatCompositionSources(item.sources), itemIndex,
|
|
)
|
|
}
|
|
envelope.imports = append(envelope.imports, item.value)
|
|
}
|
|
default:
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.%s: unknown composition field %q",
|
|
formatCompositionSources(field.value.sources), field.key, field.key,
|
|
)
|
|
}
|
|
}
|
|
root.fields = append(root.fields[:index], root.fields[index+1:]...)
|
|
for fieldIndex := range root.fields {
|
|
root.fields[fieldIndex].order = fieldIndex
|
|
}
|
|
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, envelope, nil
|
|
}
|
|
|
|
type loadedPipelineImport struct {
|
|
path string
|
|
document *compositionDocument
|
|
info os.FileInfo
|
|
}
|
|
|
|
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
|
|
if len(declared) == 0 {
|
|
return nil, nil
|
|
}
|
|
rootDir := filepath.Dir(rootPath)
|
|
rootInfo, err := os.Stat(rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
|
|
}
|
|
seenPaths := make(map[string]int, len(declared))
|
|
loaded := make([]loadedPipelineImport, 0, len(declared))
|
|
for index, raw := range declared {
|
|
if strings.TrimSpace(raw) != raw || raw == "" {
|
|
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
|
|
}
|
|
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
|
|
}
|
|
extension := filepath.Ext(filepath.FromSlash(normalized))
|
|
if extension != ".yml" && extension != ".yaml" {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
|
|
}
|
|
if prior, duplicate := seenPaths[normalized]; duplicate {
|
|
return nil, fmt.Errorf(
|
|
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
|
|
index, raw, prior,
|
|
)
|
|
}
|
|
seenPaths[normalized] = index
|
|
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
|
|
if resolved == rootPath {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
|
}
|
|
|
|
file, err := fileops.OpenConfinedRegularFile(rootDir, normalized)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
|
|
}
|
|
info, statErr := file.Stat()
|
|
if statErr != nil {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
|
|
}
|
|
if os.SameFile(rootInfo, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
|
}
|
|
for priorIndex, prior := range loaded {
|
|
if os.SameFile(prior.info, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf(
|
|
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
|
|
index, raw, priorIndex, declared[priorIndex],
|
|
)
|
|
}
|
|
}
|
|
document, parseErr := parseCompositionDocument(resolved, file)
|
|
closeErr := file.Close()
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
|
|
}
|
|
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
|
|
return nil, fmt.Errorf(
|
|
"imported configuration source %q declares composition; only the root pipeline may declare composition",
|
|
resolved,
|
|
)
|
|
}
|
|
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
|
|
}
|
|
return loaded, nil
|
|
}
|