package configkit import ( "bytes" "errors" "io" "io/fs" "os" "path" "path/filepath" "strconv" "strings" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" ) const ( maxImportDepth = 64 maxImportSources = 1024 maxGlobEntries = 1_000_000 ) type importLimits struct { depth int sources int bytes int64 nodes int entries int } func defaultImportLimits() importLimits { return importLimits{ depth: maxImportDepth, sources: maxImportSources, bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries, } } type importDeclaration struct { path string glob string optional bool location Location } type importRoute struct { canonical string locations []Location } type importTraversal struct { rootPath string root *os.Root limits importLimits rule *VersionRule documents []*document.Node sources []SourceInfo active map[string]int visited map[string]importRoute stack []importRoute usedBytes int64 usedNodes int usedEntries int } func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } root, err := os.OpenRoot(rootPath) if err != nil { diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) diagnostic.Primary = nil return nil, sourceInfos(main.source), []Diagnostic{diagnostic} } mainInfo := cloneSourceInfo(*main.source) if mainPath != "" { mainInfo.Path = mainPath } state := importTraversal{ rootPath: rootPath, root: root, limits: limits, rule: rule, documents: make([]*document.Node, 0, 1), sources: []SourceInfo{mainInfo}, active: make(map[string]int), visited: make(map[string]importRoute), usedBytes: main.bytes, usedNodes: countInputNodes(main.document), } if diagnostic := state.checkInitialLimits(); diagnostic != nil { _ = root.Close() return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} } mainIdentity := mainPath if mainIdentity == "" { mainIdentity = "" } mainRoute := importRoute{canonical: mainIdentity} state.active[mainIdentity] = 0 state.visited[mainIdentity] = mainRoute state.stack = append(state.stack, mainRoute) body, declarations, diagnostics := splitImportDirectives(main.document) if len(diagnostics) == 0 { state.documents = append(state.documents, body) diagnostics = state.walkDeclarations(basePath, declarations, 0) } closeErr := root.Close() if len(diagnostics) != 0 { return nil, cloneSourceInfos(state.sources), diagnostics } if closeErr != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} } composed, diagnostics := composeDocuments(state.documents...) return composed, cloneSourceInfos(state.sources), diagnostics } func validateFilesystemContext(source Source, context *FilesystemContext, main parsedSource) *Diagnostic { rootPath, _, _, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { return diagnostic } root, err := os.OpenRoot(rootPath) if err != nil { value := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) value.Primary = nil return &value } if err := root.Close(); err != nil { value := importDiagnostic(CodeImportRead, "configuration root could not be closed", nil) return &value } return nil } func filesystemOptionDiagnostics(source Source, context *FilesystemContext) []Diagnostic { switch source.kind { case sourceNone: return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource cannot use filesystem context")} case sourceFile: if context.BaseDir != "" { return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")} } case sourceBytes, sourceReader: if context.BaseDir == "" || context.Root == "" { return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem context requires base and root directories")} } if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) { return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem base and root directories must be absolute")} } } return nil } func importFilesystem(source Source, context *FilesystemContext, main parsedSource) (rootPath, basePath, mainPath string, diagnostic *Diagnostic) { if main.source == nil || main.document == nil { value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil) return "", "", "", &value } if source.kind == sourceFile { if context != nil && context.BaseDir != "" { value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory") return "", "", "", &value } canonical, err := canonicalExistingPath(source.path) if err != nil { value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil) return "", "", "", &value } mainPath = canonical basePath = filepath.Dir(canonical) rootPath = basePath if context != nil && context.Root != "" { rootPath, err = canonicalExistingPath(context.Root) if err != nil { value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) return "", "", "", &value } } } else { if context == nil || context.BaseDir == "" || context.Root == "" { value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories") return "", "", "", &value } if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) { value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute") return "", "", "", &value } var err error basePath, err = canonicalExistingPath(context.BaseDir) if err != nil { value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil) return "", "", "", &value } rootPath, err = canonicalExistingPath(context.Root) if err != nil { value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) return "", "", "", &value } } if !containedPath(rootPath, basePath) || (mainPath != "" && !containedPath(rootPath, mainPath)) { value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil) return "", "", "", &value } for _, candidate := range []string{rootPath, basePath} { info, err := os.Stat(candidate) if err != nil || !info.IsDir() { value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil) return "", "", "", &value } } return rootPath, basePath, mainPath, nil } func canonicalExistingPath(name string) (string, error) { absolute, err := filepath.Abs(name) if err != nil { return "", err } return filepath.EvalSymlinks(filepath.Clean(absolute)) } func containedPath(root, target string) bool { relative, err := filepath.Rel(root, target) if err != nil || filepath.IsAbs(relative) { return false } return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) } func (s *importTraversal) checkInitialLimits() *Diagnostic { if s.limits.sources < 1 { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) return &value } if s.usedBytes > s.limits.bytes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) return &value } if s.usedNodes > s.limits.nodes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) return &value } return nil } func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int) []Diagnostic { for _, declaration := range declarations { if declaration.glob != "" { matches, diagnostics := s.expandImportGlob(basePath, declaration) if len(diagnostics) != 0 { return diagnostics } if len(matches) == 0 { if declaration.optional { continue } location := declaration.location return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)} } for _, match := range matches { matched := declaration matched.path = match matched.glob = "" if diagnostics := s.visitPath(basePath, matched, depth+1); len(diagnostics) != 0 { return diagnostics } } continue } if diagnostics := s.visitPath(basePath, declaration, depth+1); len(diagnostics) != 0 { return diagnostics } } return nil } func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, depth int) []Diagnostic { location := declaration.location if depth > s.limits.depth { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} } if !validImportPath(declaration.path) { return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)} } candidate := filepath.Join(basePath, filepath.FromSlash(path.Clean(declaration.path))) if !containedPath(s.rootPath, filepath.Clean(candidate)) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} } canonical, err := canonicalExistingPath(candidate) if err != nil { if declaration.optional && errors.Is(err, fs.ErrNotExist) { return nil } code := CodeImportInvalid message := "import target could not be canonicalized" if errors.Is(err, fs.ErrNotExist) { code, message = CodeImportMissing, "required import target does not exist" } else if errors.Is(err, fs.ErrPermission) { message = "import target cannot be accessed" } return []Diagnostic{importDiagnostic(code, message, &location)} } if !containedPath(s.rootPath, canonical) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} } currentRoute := s.currentRoute(location) if activeIndex, ok := s.active[canonical]; ok { cycle := append([]importRoute(nil), s.stack[activeIndex:]...) return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} } if previous, ok := s.visited[canonical]; ok { return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} } if len(s.sources)+1 > s.limits.sources { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} } relative, err := filepath.Rel(s.rootPath, canonical) if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} } sourceID := "source-" + strconv.Itoa(len(s.sources)+1) info := SourceInfo{ID: sourceID, Label: declaration.path, Path: canonical, Layer: LayerBase, ImportedFrom: &location} s.sources = append(s.sources, info) data, skipped, diagnostic := s.readImport(relative, sourceID, declaration.optional) if diagnostic != nil { return []Diagnostic{*diagnostic} } if skipped { s.sources = s.sources[:len(s.sources)-1] return nil } root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ Depth: maxDocumentDepth, Nodes: maxDocumentNodes, }) if len(issues) != 0 { return importParseDiagnostics(sourceID, issues) } nodeCount := countInputNodes(root) if s.usedNodes+nodeCount > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } s.usedNodes += nodeCount body, declarations, diagnostics := splitImportDirectives(root) if len(diagnostics) != 0 { return diagnostics } if versionNode := fragmentVersionClaim(body, s.rule); versionNode != nil { versionLocation := publicLocation(versionNode.Location) return []Diagnostic{{ Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(), Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation, }} } route := importRoute{canonical: canonical, locations: currentRoute.locations} s.documents = append(s.documents, body) s.active[canonical] = len(s.stack) s.visited[canonical] = route s.stack = append(s.stack, route) diagnostics = s.walkDeclarations(filepath.Dir(canonical), declarations, depth) delete(s.active, canonical) s.stack = s.stack[:len(s.stack)-1] return diagnostics } func (s *importTraversal) readImport(relative, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { remaining := s.limits.bytes - s.usedBytes limit := min(int64(maxSourceBytes), remaining) if limit < 0 { limit = 0 } file, err := s.root.Open(relative) if err != nil { if optional && errors.Is(err, fs.ErrNotExist) { return nil, true, nil } value := classifySourceError(err, sourceID) return nil, false, &value } defer func() { _ = file.Close() }() info, err := file.Stat() if err != nil { value := classifySourceError(err, sourceID) return nil, false, &value } if !info.Mode().IsRegular() { value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file") return nil, false, &value } if info.Size() > limit { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID}) return nil, false, &value } data, err := io.ReadAll(io.LimitReader(file, limit+1)) if err != nil { value := classifySourceError(err, sourceID) return nil, false, &value } if int64(len(data)) > limit { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID}) return nil, false, &value } s.usedBytes += int64(len(data)) return data, false, nil } func (s *importTraversal) currentRoute(location Location) importRoute { locations := make([]Location, 0, len(s.stack)+1) for _, route := range s.stack { if len(route.locations) != 0 { locations = append(locations, route.locations[len(route.locations)-1]) } } locations = append(locations, location) return importRoute{locations: locations} } func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { body := document.Clone(root) var declarations []importDeclaration directiveIndex := -1 for index, entry := range body.Mapping { if entry.Key != "_configkit" { continue } directiveIndex = index parsed, diagnostics := parseImportDirective(entry.Value, Keys("_configkit")) if len(diagnostics) != 0 { return nil, nil, diagnostics } declarations = parsed break } if directiveIndex >= 0 { body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...) } if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 { return nil, nil, diagnostics } return body, declarations, nil } func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) { if node.Kind != document.Mapping { return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)} } var declarations []importDeclaration for _, entry := range node.Mapping { fieldPath := directivePath.Key(entry.Key) if entry.Key != "imports" { return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)} } if entry.Value.Kind != document.Sequence { return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)} } for index, item := range entry.Value.Sequence { declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index)) if len(diagnostics) != 0 { return nil, diagnostics } declarations = append(declarations, declaration) } } return declarations, nil } func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) { if node.Kind != document.Mapping { return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} } declaration := importDeclaration{location: publicLocation(node.Location)} seenTarget := false for _, entry := range node.Mapping { fieldPath := declarationPath.Key(entry.Key) switch entry.Key { case "path", "glob": if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text { return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)} } seenTarget = true declaration.location = publicLocation(entry.Value.Location) if entry.Key == "path" { declaration.path = entry.Value.Text } else { declaration.glob = entry.Value.Text } case "optional": if entry.Value.Kind != document.Boolean { return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)} } declaration.optional = entry.Value.Text == "true" default: return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} } } if !seenTarget { return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)} } return declaration, nil } func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { if node.Kind == document.Mapping { for _, entry := range node.Mapping { childPath := current.Key(entry.Key) if entry.Key == "_configkit" { location := publicLocation(entry.KeyLocation) return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: childPath, Message: "configuration directives are permitted only at the document root", Primary: &location}} } if diagnostics := nestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { return diagnostics } } } else if node.Kind == document.Sequence { for index, child := range node.Sequence { if diagnostics := nestedDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { return diagnostics } } } return nil } func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node { if rule == nil { return nil } current := root if len(rule.Path.segments) == 0 { return current } for _, segment := range rule.Path.segments { if current.Kind != document.Mapping { return current } if segment.kind == pathSegmentIndex { return nil } key := segment.key index := documentEntryIndex(current.Mapping, key) if index < 0 { return nil } current = current.Mapping[index].Value } return current } func countInputNodes(node *document.Node) int { if node == nil { return 0 } count := 1 if node.Kind == document.Mapping { count += len(node.Mapping) for _, entry := range node.Mapping { count += countInputNodes(entry.Value) } } else if node.Kind == document.Sequence { for _, child := range node.Sequence { count += countInputNodes(child) } } return count } func validImportPath(value string) bool { if value == "" || strings.TrimSpace(value) != value || path.IsAbs(value) || strings.HasPrefix(value, "//") { return false } return !hasImportScheme(value) } func hasImportScheme(value string) bool { colon := strings.IndexByte(value, ':') slash := strings.IndexByte(value, '/') if colon <= 0 || (slash >= 0 && slash < colon) { return false } for index := 0; index < colon; index++ { character := value[index] if index == 0 { if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') { return false } continue } if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && (character < '0' || character > '9') && character != '+' && character != '-' && character != '.' { return false } } return true } func sourceInfos(source *SourceInfo) []SourceInfo { if source == nil { return nil } return []SourceInfo{cloneSourceInfo(*source)} } func importParseDiagnostics(sourceID string, issues []yamladapter.Issue) []Diagnostic { diagnostics := make([]Diagnostic, len(issues)) for index, issue := range issues { diagnostics[index] = Diagnostic{ Code: Code(issue.Code), Severity: SeverityError, Message: issue.Message, Primary: &Location{SourceID: sourceID, Line: issue.Line, Column: issue.Column}, } for _, related := range issue.Related { diagnostics[index].Related = append(diagnostics[index].Related, publicLocation(related)) } } return diagnostics } func importDiagnostic(code Code, message string, location *Location) Diagnostic { return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)} } func nodeImportDiagnostic(code Code, path Path, message string, node *document.Node) Diagnostic { location := publicLocation(node.Location) return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location} } func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic { paths := make([]string, 0, len(cycle)+1) related := make([]Location, 0, len(cycle)) for _, route := range cycle { paths = append(paths, route.canonical) if len(route.locations) != 0 { related = append(related, route.locations[len(route.locations)-1]) } } paths = append(paths, target) return Diagnostic{ Code: CodeImportCycle, Severity: SeverityError, Message: "import cycle detected: " + strings.Join(paths, " -> "), Primary: &location, Related: related, } } func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic { related := append([]Location(nil), previous.locations...) if len(current.locations) > 1 { related = append(related, current.locations[:len(current.locations)-1]...) } return Diagnostic{ Code: CodeImportRepeated, Severity: SeverityError, Message: "configuration source is included by more than one import route", Primary: &location, Related: related, } }