package config import ( "fmt" "os" "path/filepath" "strings" "unicode" "gitea.maximumdirect.net/eric/narratio/internal/fileops" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" "gopkg.in/yaml.v3" ) type pipelineResolutionMetadata struct { rootPath string imports []string sources []string selectedProfile *pipelineProfileSelection effectiveDigest string ownership []pipelineFieldOwnership } type pipelineProfileSelection struct { name string source string overlayPath string } type pipelineFieldOwnership struct { path string sources []string } type pipelineCompositionEnvelope struct { imports []string defaultProfile *string profiles []pipelineProfileDeclaration } type pipelineProfileDeclaration struct { name string overlay string } func loadComposedPipeline(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { rootPath, err := filepath.Abs(path) if err != nil { return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err) } rootFile, err := os.Open(rootPath) if err != nil { return nil, fmt.Errorf("pipeline file %q: open: %w", path, err) } rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile) closeErr := rootFile.Close() if parseErr != nil { return nil, parseErr } if closeErr != nil { return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr) } baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument) if err != nil { return nil, err } imports, err := loadPipelineImports(rootPath, envelope.imports) if err != nil { return nil, err } selection, err := selectPipelineProfile(envelope, opts) if err != nil { return nil, err } overlays, err := loadPipelineProfileOverlays(rootPath, envelope.profiles, imports) if err != nil { return nil, err } documents := make([]*compositionDocument, 0, len(imports)+1) documents = append(documents, baseRoot) for _, imported := range imports { documents = append(documents, imported.document) } merged, err := mergeAdditiveCompositions(documents...) if err != nil { return nil, err } if selection != nil { overlay, ok := loadedProfileOverlay(overlays, selection.name) if !ok { return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name) } merged, err = mergeOverlayComposition(merged, overlay.document) if err != nil { return nil, err } selection.overlayPath = overlay.path } rendered, err := merged.canonicalYAML() if err != nil { return nil, err } var cfg PipelineConfig if err := decodeStrictYAMLFromReader("pipeline", rootPath, strings.NewReader(string(rendered)), &cfg); err != nil { return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err) } records, err := merged.semanticRecords() if err != nil { return nil, err } metadata := &pipelineResolutionMetadata{ rootPath: rootPath, sources: append([]string(nil), merged.sources...), selectedProfile: selection, } for _, imported := range imports { metadata.imports = append(metadata.imports, imported.path) } for _, record := range records { metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{ path: record.Path, sources: append([]string(nil), record.Sources...), }) } cfg.resolution = metadata return &cfg, nil } func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) { if err := validateCompositionDocument(document, "root pipeline"); err != nil { return nil, pipelineCompositionEnvelope{}, err } root := cloneCompositionNode(document.root) index := compositionFieldIndex(root.fields, "composition") if index < 0 { return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, pipelineCompositionEnvelope{}, nil } envelopeNode := root.fields[index].value if envelopeNode.kind != yaml.MappingNode { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition: expected a mapping, got %s", formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind), ) } var envelope pipelineCompositionEnvelope for _, field := range envelopeNode.fields { switch field.key { case "imports": if field.value.kind != yaml.SequenceNode { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.imports: expected a list, got %s", formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), ) } for itemIndex, item := range field.value.items { if item.kind != yaml.ScalarNode || item.tag != "!!str" { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.imports[%d]: expected a string path", formatCompositionSources(item.sources), itemIndex, ) } envelope.imports = append(envelope.imports, item.value) } case "default_profile": if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.default_profile: expected a string profile name", formatCompositionSources(field.value.sources), ) } name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile") if err != nil { return nil, pipelineCompositionEnvelope{}, err } envelope.defaultProfile = &name case "profiles": if field.value.kind != yaml.MappingNode { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.profiles: expected a mapping, got %s", formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), ) } if len(field.value.fields) == 0 { return nil, pipelineCompositionEnvelope{}, fmt.Errorf("composition.profiles must declare at least one named profile") } for _, profileField := range field.value.fields { name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name") if err != nil { return nil, pipelineCompositionEnvelope{}, err } profile, err := parsePipelineProfileDeclaration(name, profileField.value) if err != nil { return nil, pipelineCompositionEnvelope{}, err } envelope.profiles = append(envelope.profiles, profile) } default: return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.%s: unknown composition field %q", formatCompositionSources(field.value.sources), field.key, field.key, ) } } if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "composition.default_profile %q does not name a declared profile", *envelope.defaultProfile, ) } root.fields = append(root.fields[:index], root.fields[index+1:]...) for fieldIndex := range root.fields { root.fields[fieldIndex].order = fieldIndex } return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, envelope, nil } func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) { path := "composition.profiles." + name if node.kind != yaml.MappingNode { return pipelineProfileDeclaration{}, fmt.Errorf( "configuration source %s at %s: expected a mapping, got %s", formatCompositionSources(node.sources), path, yamlKindName(node.kind), ) } profile := pipelineProfileDeclaration{name: name} for _, field := range node.fields { if field.key != "overlay" { return pipelineProfileDeclaration{}, fmt.Errorf( "configuration source %s at %s.%s: unknown profile field %q; only overlay is supported", formatCompositionSources(field.value.sources), path, field.key, field.key, ) } if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { return pipelineProfileDeclaration{}, fmt.Errorf( "configuration source %s at %s.overlay: expected a string path", formatCompositionSources(field.value.sources), path, ) } profile.overlay = field.value.value } if profile.overlay == "" { return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path) } return profile, nil } func normalizePipelineProfileName(value, label string) (string, error) { if value == "" || strings.TrimSpace(value) != value { return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label) } for _, character := range value { if unicode.IsControl(character) { return "", fmt.Errorf("%s %q must not contain control characters", label, value) } } return value, nil } func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool { for _, profile := range profiles { if profile.name == name { return true } } return false } func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) { if opts.Profile != nil { name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection") if err != nil { return nil, err } if len(envelope.profiles) == 0 { return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name) } if !pipelineProfileDeclared(envelope.profiles, name) { return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name) } return &pipelineProfileSelection{name: name, source: "cli"}, nil } if len(envelope.profiles) == 0 { return nil, nil } if envelope.defaultProfile == nil { return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected") } return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil } type loadedPipelineImport struct { path string document *compositionDocument info os.FileInfo } type loadedPipelineProfileOverlay struct { profile string path string document *compositionDocument info os.FileInfo } func loadPipelineProfileOverlays( rootPath string, declared []pipelineProfileDeclaration, imports []loadedPipelineImport, ) ([]loadedPipelineProfileOverlay, error) { if len(declared) == 0 { return nil, nil } rootDir := filepath.Dir(rootPath) rootInfo, err := os.Stat(rootPath) if err != nil { return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) } seenPaths := make(map[string]string, len(declared)) loaded := make([]loadedPipelineProfileOverlay, 0, len(declared)) for _, profile := range declared { label := "composition.profiles." + profile.name + ".overlay" raw := profile.overlay if strings.TrimSpace(raw) != raw || raw == "" { return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label) } normalized, err := pathsafe.NormalizeRelativeDestination(raw) if err != nil { return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err) } extension := filepath.Ext(filepath.FromSlash(normalized)) if extension != ".yml" && extension != ".yaml" { return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw) } if prior, duplicate := seenPaths[normalized]; duplicate { return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior) } seenPaths[normalized] = profile.name resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) file, err := fileops.OpenConfinedRegularFile(rootDir, normalized) if err != nil { return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err) } info, statErr := file.Stat() if statErr != nil { _ = file.Close() return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr) } if os.SameFile(rootInfo, info) { _ = file.Close() return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw) } for _, imported := range imports { if os.SameFile(imported.info, info) { _ = file.Close() return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path) } } for _, prior := range loaded { if os.SameFile(prior.info, info) { _ = file.Close() return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile) } } document, parseErr := parseCompositionDocument(resolved, file) closeErr := file.Close() if parseErr != nil { return nil, parseErr } if closeErr != nil { return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr) } if compositionFieldIndex(document.root.fields, "composition") >= 0 { return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved) } loaded = append(loaded, loadedPipelineProfileOverlay{ profile: profile.name, path: resolved, document: document, info: info, }) } return loaded, nil } func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) { for _, overlay := range overlays { if overlay.profile == name { return overlay, true } } return loadedPipelineProfileOverlay{}, false } func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) { if len(declared) == 0 { return nil, nil } rootDir := filepath.Dir(rootPath) rootInfo, err := os.Stat(rootPath) if err != nil { return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) } seenPaths := make(map[string]int, len(declared)) loaded := make([]loadedPipelineImport, 0, len(declared)) for index, raw := range declared { if strings.TrimSpace(raw) != raw || raw == "" { return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index) } normalized, err := pathsafe.NormalizeRelativeDestination(raw) if err != nil { return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err) } extension := filepath.Ext(filepath.FromSlash(normalized)) if extension != ".yml" && extension != ".yaml" { return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw) } if prior, duplicate := seenPaths[normalized]; duplicate { return nil, fmt.Errorf( "composition.imports[%d] path %q duplicates composition.imports[%d] after normalization", index, raw, prior, ) } seenPaths[normalized] = index resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) if resolved == rootPath { return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) } file, err := fileops.OpenConfinedRegularFile(rootDir, normalized) if err != nil { return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err) } info, statErr := file.Stat() if statErr != nil { _ = file.Close() return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr) } if os.SameFile(rootInfo, info) { _ = file.Close() return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) } for priorIndex, prior := range loaded { if os.SameFile(prior.info, info) { _ = file.Close() return nil, fmt.Errorf( "composition.imports[%d] path %q references the same file as composition.imports[%d] %q", index, raw, priorIndex, declared[priorIndex], ) } } document, parseErr := parseCompositionDocument(resolved, file) closeErr := file.Close() if parseErr != nil { return nil, parseErr } if closeErr != nil { return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr) } if compositionFieldIndex(document.root.fields, "composition") >= 0 { return nil, fmt.Errorf( "imported configuration source %q declares composition; only the root pipeline may declare composition", resolved, ) } loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info}) } return loaded, nil }