package config import ( "fmt" "os" "path/filepath" "strings" "gitea.maximumdirect.net/eric/narratio/internal/fileops" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" "gopkg.in/yaml.v3" ) type pipelineResolutionMetadata struct { rootPath string imports []string sources []string ownership []pipelineFieldOwnership } type pipelineFieldOwnership struct { path string sources []string } type pipelineCompositionEnvelope struct { imports []string } func loadComposedPipeline(path string) (*PipelineConfig, error) { rootPath, err := filepath.Abs(path) if err != nil { return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err) } rootFile, err := os.Open(rootPath) if err != nil { return nil, fmt.Errorf("pipeline file %q: open: %w", path, err) } rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile) closeErr := rootFile.Close() if parseErr != nil { return nil, parseErr } if closeErr != nil { return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr) } baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument) if err != nil { return nil, err } imports, err := loadPipelineImports(rootPath, envelope.imports) if err != nil { return nil, err } documents := make([]*compositionDocument, 0, len(imports)+1) documents = append(documents, baseRoot) for _, imported := range imports { documents = append(documents, imported.document) } merged, err := mergeAdditiveCompositions(documents...) if err != nil { return nil, err } rendered, err := merged.canonicalYAML() if err != nil { return nil, err } var cfg PipelineConfig if err := decodeStrictYAMLFromReader("pipeline", rootPath, strings.NewReader(string(rendered)), &cfg); err != nil { return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err) } records, err := merged.semanticRecords() if err != nil { return nil, err } metadata := &pipelineResolutionMetadata{ rootPath: rootPath, sources: append([]string(nil), merged.sources...), } for _, imported := range imports { metadata.imports = append(metadata.imports, imported.path) } for _, record := range records { metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{ path: record.Path, sources: append([]string(nil), record.Sources...), }) } cfg.resolution = metadata return &cfg, nil } func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) { if err := validateCompositionDocument(document, "root pipeline"); err != nil { return nil, pipelineCompositionEnvelope{}, err } root := cloneCompositionNode(document.root) index := compositionFieldIndex(root.fields, "composition") if index < 0 { return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, pipelineCompositionEnvelope{}, nil } envelopeNode := root.fields[index].value if envelopeNode.kind != yaml.MappingNode { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition: expected a mapping, got %s", formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind), ) } var envelope pipelineCompositionEnvelope for _, field := range envelopeNode.fields { switch field.key { case "imports": if field.value.kind != yaml.SequenceNode { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.imports: expected a list, got %s", formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), ) } for itemIndex, item := range field.value.items { if item.kind != yaml.ScalarNode || item.tag != "!!str" { return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.imports[%d]: expected a string path", formatCompositionSources(item.sources), itemIndex, ) } envelope.imports = append(envelope.imports, item.value) } default: return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.%s: unknown composition field %q", formatCompositionSources(field.value.sources), field.key, field.key, ) } } root.fields = append(root.fields[:index], root.fields[index+1:]...) for fieldIndex := range root.fields { root.fields[fieldIndex].order = fieldIndex } return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, envelope, nil } type loadedPipelineImport struct { path string document *compositionDocument info os.FileInfo } func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) { if len(declared) == 0 { return nil, nil } rootDir := filepath.Dir(rootPath) rootInfo, err := os.Stat(rootPath) if err != nil { return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) } seenPaths := make(map[string]int, len(declared)) loaded := make([]loadedPipelineImport, 0, len(declared)) for index, raw := range declared { if strings.TrimSpace(raw) != raw || raw == "" { return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index) } normalized, err := pathsafe.NormalizeRelativeDestination(raw) if err != nil { return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err) } extension := filepath.Ext(filepath.FromSlash(normalized)) if extension != ".yml" && extension != ".yaml" { return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw) } if prior, duplicate := seenPaths[normalized]; duplicate { return nil, fmt.Errorf( "composition.imports[%d] path %q duplicates composition.imports[%d] after normalization", index, raw, prior, ) } seenPaths[normalized] = index resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) if resolved == rootPath { return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) } file, err := fileops.OpenConfinedRegularFile(rootDir, normalized) if err != nil { return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err) } info, statErr := file.Stat() if statErr != nil { _ = file.Close() return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr) } if os.SameFile(rootInfo, info) { _ = file.Close() return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) } for priorIndex, prior := range loaded { if os.SameFile(prior.info, info) { _ = file.Close() return nil, fmt.Errorf( "composition.imports[%d] path %q references the same file as composition.imports[%d] %q", index, raw, priorIndex, declared[priorIndex], ) } } document, parseErr := parseCompositionDocument(resolved, file) closeErr := file.Close() if parseErr != nil { return nil, parseErr } if closeErr != nil { return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr) } if compositionFieldIndex(document.root.fields, "composition") >= 0 { return nil, fmt.Errorf( "imported configuration source %q declares composition; only the root pipeline may declare composition", resolved, ) } loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info}) } return loaded, nil }