513 lines
18 KiB
Go
513 lines
18 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type pipelineResolutionMetadata struct {
|
|
rootPath string
|
|
imports []string
|
|
sources []string
|
|
selectedProfile *pipelineProfileSelection
|
|
effectiveDigest string
|
|
ownership []pipelineFieldOwnership
|
|
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
|
artifactFamiliesExpanded bool
|
|
publishDeclared bool
|
|
}
|
|
|
|
type pipelineProfileSelection struct {
|
|
name string
|
|
source string
|
|
overlayPath string
|
|
}
|
|
|
|
type pipelineFieldOwnership struct {
|
|
path string
|
|
sources []string
|
|
}
|
|
|
|
type pipelineCompositionEnvelope struct {
|
|
imports []string
|
|
defaultProfile *string
|
|
profiles []pipelineProfileDeclaration
|
|
}
|
|
|
|
type pipelineProfileDeclaration struct {
|
|
name string
|
|
overlay string
|
|
}
|
|
|
|
// pipelineCompositionSources retains one validated root source set. Each
|
|
// selected profile is resolved from a cloned base document so callers can
|
|
// safely compare or otherwise resolve multiple profiles without rereading or
|
|
// mutating the source set.
|
|
type pipelineCompositionSources struct {
|
|
rootPath string
|
|
envelope pipelineCompositionEnvelope
|
|
imports []loadedPipelineImport
|
|
overlays []loadedPipelineProfileOverlay
|
|
overlaysLoaded bool
|
|
base *compositionDocument
|
|
}
|
|
|
|
func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) {
|
|
rootPath, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
|
|
}
|
|
rootFile, err := os.Open(rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
|
|
}
|
|
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
|
|
closeErr := rootFile.Close()
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
|
|
}
|
|
|
|
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
imports, err := loadPipelineImports(rootPath, envelope.imports)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
documents := make([]*compositionDocument, 0, len(imports)+1)
|
|
documents = append(documents, baseRoot)
|
|
for _, imported := range imports {
|
|
documents = append(documents, imported.document)
|
|
}
|
|
merged, err := mergeAdditiveCompositions(documents...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &pipelineCompositionSources{
|
|
rootPath: rootPath,
|
|
envelope: envelope,
|
|
imports: imports,
|
|
base: merged,
|
|
}, nil
|
|
}
|
|
|
|
func (sources *pipelineCompositionSources) loadOverlays() error {
|
|
if sources == nil {
|
|
return fmt.Errorf("pipeline composition sources are required")
|
|
}
|
|
if sources.overlaysLoaded {
|
|
return nil
|
|
}
|
|
overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sources.overlays = overlays
|
|
sources.overlaysLoaded = true
|
|
return nil
|
|
}
|
|
|
|
func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) {
|
|
if sources == nil || sources.base == nil {
|
|
return nil, fmt.Errorf("pipeline composition sources are required")
|
|
}
|
|
if !sources.overlaysLoaded {
|
|
return nil, fmt.Errorf("pipeline profile overlays have not been loaded")
|
|
}
|
|
selection, err := selectPipelineProfile(sources.envelope, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
merged := &compositionDocument{
|
|
root: cloneCompositionNode(sources.base.root),
|
|
sources: append([]string(nil), sources.base.sources...),
|
|
}
|
|
if selection != nil {
|
|
overlay, ok := loadedProfileOverlay(sources.overlays, selection.name)
|
|
if !ok {
|
|
return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name)
|
|
}
|
|
merged, err = mergeOverlayComposition(merged, overlay.document)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
selection.overlayPath = overlay.path
|
|
}
|
|
|
|
rendered, err := merged.canonicalYAML()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var cfg PipelineConfig
|
|
if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
|
|
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
|
|
}
|
|
records, err := merged.semanticRecords()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
metadata := &pipelineResolutionMetadata{
|
|
rootPath: sources.rootPath,
|
|
sources: append([]string(nil), merged.sources...),
|
|
selectedProfile: selection,
|
|
}
|
|
for _, imported := range sources.imports {
|
|
metadata.imports = append(metadata.imports, imported.path)
|
|
}
|
|
for _, record := range records {
|
|
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
|
|
path: record.Path, sources: append([]string(nil), record.Sources...),
|
|
})
|
|
}
|
|
cfg.resolution = metadata
|
|
return &cfg, nil
|
|
}
|
|
|
|
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
|
|
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
|
|
return nil, pipelineCompositionEnvelope{}, err
|
|
}
|
|
root := cloneCompositionNode(document.root)
|
|
index := compositionFieldIndex(root.fields, "composition")
|
|
if index < 0 {
|
|
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, pipelineCompositionEnvelope{}, nil
|
|
}
|
|
envelopeNode := root.fields[index].value
|
|
if envelopeNode.kind != yaml.MappingNode {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition: expected a mapping, got %s",
|
|
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
|
|
)
|
|
}
|
|
|
|
var envelope pipelineCompositionEnvelope
|
|
for _, field := range envelopeNode.fields {
|
|
switch field.key {
|
|
case "imports":
|
|
if field.value.kind != yaml.SequenceNode {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.imports: expected a list, got %s",
|
|
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
|
|
)
|
|
}
|
|
for itemIndex, item := range field.value.items {
|
|
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.imports[%d]: expected a string path",
|
|
formatCompositionSources(item.sources), itemIndex,
|
|
)
|
|
}
|
|
envelope.imports = append(envelope.imports, item.value)
|
|
}
|
|
case "default_profile":
|
|
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.default_profile: expected a string profile name",
|
|
formatCompositionSources(field.value.sources),
|
|
)
|
|
}
|
|
name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile")
|
|
if err != nil {
|
|
return nil, pipelineCompositionEnvelope{}, err
|
|
}
|
|
envelope.defaultProfile = &name
|
|
case "profiles":
|
|
if field.value.kind != yaml.MappingNode {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.profiles: expected a mapping, got %s",
|
|
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
|
|
)
|
|
}
|
|
if len(field.value.fields) == 0 {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf("composition.profiles must declare at least one named profile")
|
|
}
|
|
for _, profileField := range field.value.fields {
|
|
name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name")
|
|
if err != nil {
|
|
return nil, pipelineCompositionEnvelope{}, err
|
|
}
|
|
profile, err := parsePipelineProfileDeclaration(name, profileField.value)
|
|
if err != nil {
|
|
return nil, pipelineCompositionEnvelope{}, err
|
|
}
|
|
envelope.profiles = append(envelope.profiles, profile)
|
|
}
|
|
default:
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"configuration source %s at composition.%s: unknown composition field %q",
|
|
formatCompositionSources(field.value.sources), field.key, field.key,
|
|
)
|
|
}
|
|
}
|
|
if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) {
|
|
return nil, pipelineCompositionEnvelope{}, fmt.Errorf(
|
|
"composition.default_profile %q does not name a declared profile", *envelope.defaultProfile,
|
|
)
|
|
}
|
|
root.fields = append(root.fields[:index], root.fields[index+1:]...)
|
|
for fieldIndex := range root.fields {
|
|
root.fields[fieldIndex].order = fieldIndex
|
|
}
|
|
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, envelope, nil
|
|
}
|
|
|
|
func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) {
|
|
path := "composition.profiles." + name
|
|
if node.kind != yaml.MappingNode {
|
|
return pipelineProfileDeclaration{}, fmt.Errorf(
|
|
"configuration source %s at %s: expected a mapping, got %s",
|
|
formatCompositionSources(node.sources), path, yamlKindName(node.kind),
|
|
)
|
|
}
|
|
profile := pipelineProfileDeclaration{name: name}
|
|
for _, field := range node.fields {
|
|
if field.key != "overlay" {
|
|
return pipelineProfileDeclaration{}, fmt.Errorf(
|
|
"configuration source %s at %s.%s: unknown profile field %q; only overlay is supported",
|
|
formatCompositionSources(field.value.sources), path, field.key, field.key,
|
|
)
|
|
}
|
|
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
|
|
return pipelineProfileDeclaration{}, fmt.Errorf(
|
|
"configuration source %s at %s.overlay: expected a string path",
|
|
formatCompositionSources(field.value.sources), path,
|
|
)
|
|
}
|
|
profile.overlay = field.value.value
|
|
}
|
|
if profile.overlay == "" {
|
|
return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path)
|
|
}
|
|
return profile, nil
|
|
}
|
|
|
|
func normalizePipelineProfileName(value, label string) (string, error) {
|
|
if value == "" || strings.TrimSpace(value) != value {
|
|
return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label)
|
|
}
|
|
for _, character := range value {
|
|
if unicode.IsControl(character) {
|
|
return "", fmt.Errorf("%s %q must not contain control characters", label, value)
|
|
}
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool {
|
|
for _, profile := range profiles {
|
|
if profile.name == name {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) {
|
|
if opts.Profile != nil {
|
|
name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(envelope.profiles) == 0 {
|
|
return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name)
|
|
}
|
|
if !pipelineProfileDeclared(envelope.profiles, name) {
|
|
return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name)
|
|
}
|
|
return &pipelineProfileSelection{name: name, source: "cli"}, nil
|
|
}
|
|
if len(envelope.profiles) == 0 {
|
|
return nil, nil
|
|
}
|
|
if envelope.defaultProfile == nil {
|
|
return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected")
|
|
}
|
|
return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil
|
|
}
|
|
|
|
type loadedPipelineImport struct {
|
|
path string
|
|
document *compositionDocument
|
|
info os.FileInfo
|
|
}
|
|
|
|
type loadedPipelineProfileOverlay struct {
|
|
profile string
|
|
path string
|
|
document *compositionDocument
|
|
info os.FileInfo
|
|
}
|
|
|
|
func loadPipelineProfileOverlays(
|
|
rootPath string,
|
|
declared []pipelineProfileDeclaration,
|
|
imports []loadedPipelineImport,
|
|
) ([]loadedPipelineProfileOverlay, error) {
|
|
if len(declared) == 0 {
|
|
return nil, nil
|
|
}
|
|
rootDir := filepath.Dir(rootPath)
|
|
rootInfo, err := os.Stat(rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
|
|
}
|
|
seenPaths := make(map[string]string, len(declared))
|
|
loaded := make([]loadedPipelineProfileOverlay, 0, len(declared))
|
|
for _, profile := range declared {
|
|
label := "composition.profiles." + profile.name + ".overlay"
|
|
raw := profile.overlay
|
|
if strings.TrimSpace(raw) != raw || raw == "" {
|
|
return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label)
|
|
}
|
|
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err)
|
|
}
|
|
extension := filepath.Ext(filepath.FromSlash(normalized))
|
|
if extension != ".yml" && extension != ".yaml" {
|
|
return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw)
|
|
}
|
|
if prior, duplicate := seenPaths[normalized]; duplicate {
|
|
return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior)
|
|
}
|
|
seenPaths[normalized] = profile.name
|
|
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
|
|
file, err := fileops.OpenConfinedRegularFile(rootDir, normalized)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err)
|
|
}
|
|
info, statErr := file.Stat()
|
|
if statErr != nil {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr)
|
|
}
|
|
if os.SameFile(rootInfo, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw)
|
|
}
|
|
for _, imported := range imports {
|
|
if os.SameFile(imported.info, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path)
|
|
}
|
|
}
|
|
for _, prior := range loaded {
|
|
if os.SameFile(prior.info, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile)
|
|
}
|
|
}
|
|
document, parseErr := parseCompositionDocument(resolved, file)
|
|
closeErr := file.Close()
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr)
|
|
}
|
|
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
|
|
return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved)
|
|
}
|
|
loaded = append(loaded, loadedPipelineProfileOverlay{
|
|
profile: profile.name, path: resolved, document: document, info: info,
|
|
})
|
|
}
|
|
return loaded, nil
|
|
}
|
|
|
|
func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) {
|
|
for _, overlay := range overlays {
|
|
if overlay.profile == name {
|
|
return overlay, true
|
|
}
|
|
}
|
|
return loadedPipelineProfileOverlay{}, false
|
|
}
|
|
|
|
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
|
|
if len(declared) == 0 {
|
|
return nil, nil
|
|
}
|
|
rootDir := filepath.Dir(rootPath)
|
|
rootInfo, err := os.Stat(rootPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
|
|
}
|
|
seenPaths := make(map[string]int, len(declared))
|
|
loaded := make([]loadedPipelineImport, 0, len(declared))
|
|
for index, raw := range declared {
|
|
if strings.TrimSpace(raw) != raw || raw == "" {
|
|
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
|
|
}
|
|
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
|
|
}
|
|
extension := filepath.Ext(filepath.FromSlash(normalized))
|
|
if extension != ".yml" && extension != ".yaml" {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
|
|
}
|
|
if prior, duplicate := seenPaths[normalized]; duplicate {
|
|
return nil, fmt.Errorf(
|
|
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
|
|
index, raw, prior,
|
|
)
|
|
}
|
|
seenPaths[normalized] = index
|
|
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
|
|
if resolved == rootPath {
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
|
}
|
|
|
|
file, err := fileops.OpenConfinedRegularFile(rootDir, normalized)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
|
|
}
|
|
info, statErr := file.Stat()
|
|
if statErr != nil {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
|
|
}
|
|
if os.SameFile(rootInfo, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
|
}
|
|
for priorIndex, prior := range loaded {
|
|
if os.SameFile(prior.info, info) {
|
|
_ = file.Close()
|
|
return nil, fmt.Errorf(
|
|
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
|
|
index, raw, priorIndex, declared[priorIndex],
|
|
)
|
|
}
|
|
}
|
|
document, parseErr := parseCompositionDocument(resolved, file)
|
|
closeErr := file.Close()
|
|
if parseErr != nil {
|
|
return nil, parseErr
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
|
|
}
|
|
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
|
|
return nil, fmt.Errorf(
|
|
"imported configuration source %q declares composition; only the root pipeline may declare composition",
|
|
resolved,
|
|
)
|
|
}
|
|
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
|
|
}
|
|
return loaded, nil
|
|
}
|