package engine import ( "bytes" "errors" "io/fs" "strconv" "strings" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" ) const ( maxImportDepth = 64 maxImportSources = 1024 maxGlobEntries = 1_000_000 ) type importLimits struct { depth int sources int bytes int64 nodes int entries int } func defaultImportLimits() importLimits { return importLimits{ depth: maxImportDepth, sources: maxImportSources, bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries, } } type importRoute struct { canonical string locations []Location } type importAccounting struct { sources int bytes int64 nodes int entries int } type importTraversal struct { access *filesystemAccess accounting *importAccounting limits importLimits rule *VersionRule versionState *sourceVersionState scope acquisitionScope sources []SourceInfo active map[resourceIdentity]int visited map[resourceIdentity]importRoute stack []importRoute } func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedDocumentsTracked(source, context, main, Path{}, rule, limits, &importAccounting{}, nil, nil) } func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedDocumentsTrackedWithVersion(source, context, main, destination, rule, limits, accounting, visited, hostToken, nil) } func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { var processing *sourceVersionProcessingResult if versionState != nil && versionState.mode != sourceVersionModeDefault { processing = versionState.mainProcessing } return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing) } func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) { if nativeContext(source, context) { access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } access, err := openHostAccess(rootPath) if err != nil { diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) diagnostic.Primary = nil return nil, sourceInfos(main.source), []Diagnostic{diagnostic} } if hostToken != nil { access.token = hostToken } return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTracked(access, basePath, mainPath, main, Path{}, rule, limits, accounting, nil) } func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, nil) } func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { var processing *sourceVersionProcessingResult if versionState != nil && versionState.mode != sourceVersionModeDefault { processing = versionState.mainProcessing } return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) { if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} } for _, directory := range []string{access.rootPath, basePath} { info, err := access.stat(directory) if err != nil || !info.IsDir() { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)} } } mainInfo := cloneSourceInfo(*main.source) mainInfo.Destination = destination.clone() if mainPath != "" { mainInfo.Path = mainPath } mainKey := access.identity(mainPath) if mainPath == "" { mainKey = access.syntheticIdentity() } if previous, repeated := visited[mainKey]; repeated { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})} } accounting.sources++ accounting.bytes += main.bytes accounting.nodes += countInputNodes(main.document) state := importTraversal{ access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState, scope: acquisitionScope{destination: destination.clone()}, sources: []SourceInfo{mainInfo}, active: make(map[resourceIdentity]int), visited: visited, } if state.visited == nil { state.visited = make(map[resourceIdentity]importRoute) } if diagnostic := state.checkInitialLimits(); diagnostic != nil { _ = access.close() return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} } mainIdentity := mainPath if mainIdentity == "" { mainIdentity = "" } mainRoute := importRoute{canonical: mainIdentity} state.active[mainKey] = 0 state.visited[mainKey] = mainRoute state.stack = append(state.stack, mainRoute) var body *document.Node var declarations []importDeclaration var diagnostics []Diagnostic if processing != nil { body = processing.payload declarations = processing.imports } else { body, declarations, diagnostics = splitImportDirectives(main.document) } if len(diagnostics) == 0 { _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) placementNodes := destinationWrapperNodeCount(destination) if err != nil { diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} } else if accounting.nodes+placementNodes > limits.nodes { diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} } else { accounting.nodes += placementNodes state.scope.contributions = append(state.scope.contributions, acquiredContribution{ source: cloneSourceInfo(mainInfo), payload: body, document: placed, destination: destination.clone(), processing: processing, }) diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) } } closeErr := access.close() if len(diagnostics) != 0 { return nil, cloneSourceInfos(state.sources), diagnostics } if closeErr != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} } documents := make([]*document.Node, 0, len(state.scope.contributions)) for _, contribution := range state.scope.contributions { documents = append(documents, contribution.document) } composed, diagnostics := composeDocuments(documents...) if composed != nil { composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)} composed.inputBytes = accounting.bytes composed.inputNodes = accounting.nodes } return composed, cloneSourceInfos(state.sources), diagnostics } func (s *importTraversal) checkInitialLimits() *Diagnostic { if s.accounting.sources > s.limits.sources { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) return &value } if s.accounting.bytes > s.limits.bytes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) return &value } if s.accounting.nodes > s.limits.nodes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) return &value } return nil } func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int, parentDestination Path) []Diagnostic { for _, declaration := range declarations { childDestination := appendDestination(parentDestination, declaration.destination) if declaration.glob != "" { matches, diagnostics := s.expandImportGlob(basePath, declaration) if len(diagnostics) != 0 { return diagnostics } if len(matches) == 0 { if declaration.optional { continue } location := declaration.location return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)} } for _, match := range matches { matched := declaration matched.path = match matched.glob = "" if diagnostics := s.visitPath(basePath, matched, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 { return diagnostics } } continue } if diagnostics := s.visitPath(basePath, declaration, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 { return diagnostics } } return nil } func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic { location := declaration.location if depth > s.limits.depth { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} } if !validImportPath(declaration.path) { return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)} } candidate := s.access.join(basePath, declaration.path) if !s.access.contains(candidate) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} } canonical, err := s.access.canonical(candidate) if err != nil { if declaration.optional && errors.Is(err, fs.ErrNotExist) { return nil } code := CodeImportInvalid message := "import target could not be canonicalized" if errors.Is(err, fs.ErrNotExist) { code, message = CodeImportMissing, "required import target does not exist" } else if errors.Is(err, fs.ErrPermission) { message = "import target cannot be accessed" } return []Diagnostic{importDiagnostic(code, message, &location)} } if !s.access.contains(canonical) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} } currentRoute := s.currentRoute(location) identity := s.access.identity(canonical) if activeIndex, ok := s.active[identity]; ok { cycle := append([]importRoute(nil), s.stack[activeIndex:]...) return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} } if previous, ok := s.visited[identity]; ok { return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} } if s.accounting.sources+1 > s.limits.sources { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} } if _, err := s.access.relative(canonical); err != nil { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} } sourceID := "source-" + strconv.Itoa(s.accounting.sources+1) kind := SourceKindFile label := declaration.path if s.access.native != nil { kind = SourceKindFilesystem label = s.access.label + "/" + canonical } info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: destination.clone(), Layer: LayerBase, Kind: kind, ImportedFrom: &location} if kind == SourceKindFilesystem { info.namespace = s.access.token } s.sources = append(s.sources, info) data, skipped, diagnostic := s.readImport(canonical, sourceID, declaration.optional) if diagnostic != nil { return []Diagnostic{*diagnostic} } if skipped { s.sources = s.sources[:len(s.sources)-1] return nil } s.accounting.sources++ root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ Depth: maxDocumentDepth, Nodes: maxDocumentNodes, }) if len(issues) != 0 { return parserDiagnostics(sourceID, issues) } nodeCount := countInputNodes(root) if s.accounting.nodes+nodeCount > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } s.accounting.nodes += nodeCount var body *document.Node var declarations []importDeclaration var diagnostics []Diagnostic var processing *sourceVersionProcessingResult if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault { processed, failures := s.versionState.process(root, sourceVersionContribution, destination, sourceID) if len(failures) != 0 { return failures } processing = &processed body, declarations = processed.payload, processed.imports } else { body, declarations, diagnostics = splitImportDirectives(root) if len(diagnostics) != 0 { return diagnostics } } _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} } placementNodes := destinationWrapperNodeCount(destination) if s.accounting.nodes+placementNodes > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } s.accounting.nodes += placementNodes protectionRule := s.rule if s.versionState != nil && s.versionState.effectiveRule != nil { protectionRule = s.versionState.effectiveRule } if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil { versionLocation := publicLocation(versionNode.Location) return []Diagnostic{{ Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(), Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation, }} } route := importRoute{canonical: canonical, locations: currentRoute.locations} s.scope.contributions = append(s.scope.contributions, acquiredContribution{ source: cloneSourceInfo(info), payload: body, document: placed, destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, }) s.active[identity] = len(s.stack) s.visited[identity] = route s.stack = append(s.stack, route) diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth, destination) delete(s.active, identity) s.stack = s.stack[:len(s.stack)-1] return diagnostics } func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { remaining := s.limits.bytes - s.accounting.bytes limit := min(int64(maxSourceBytes), remaining) if limit < 0 { limit = 0 } data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) if failure != nil { if optional && failure.missingTarget { return nil, true, nil } value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") return nil, false, &value } s.accounting.bytes += int64(len(data)) return data, false, nil } func (s *importTraversal) currentRoute(location Location) importRoute { locations := make([]Location, 0, len(s.stack)+1) for _, route := range s.stack { if len(route.locations) != 0 { locations = append(locations, route.locations[len(route.locations)-1]) } } locations = append(locations, location) return importRoute{locations: locations} } func sourceInfos(source *SourceInfo) []SourceInfo { if source == nil { return nil } return []SourceInfo{cloneSourceInfo(*source)} } func importDiagnostic(code Code, message string, location *Location) Diagnostic { return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)} } func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic { paths := make([]string, 0, len(cycle)+1) related := make([]Location, 0, len(cycle)) for _, route := range cycle { paths = append(paths, route.canonical) if len(route.locations) != 0 { related = append(related, route.locations[len(route.locations)-1]) } } paths = append(paths, target) return Diagnostic{ Code: CodeImportCycle, Severity: SeverityError, Message: "import cycle detected: " + strings.Join(paths, " -> "), Primary: &location, Related: related, } } func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic { related := append([]Location(nil), previous.locations...) if len(current.locations) > 1 { related = append(related, current.locations[:len(current.locations)-1]...) } return Diagnostic{ Code: CodeImportRepeated, Severity: SeverityError, Message: "configuration source is included by more than one import route", Primary: &location, Related: related, } }