package engine import ( "bytes" "errors" "io/fs" "strconv" "strings" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" ) const ( maxImportDepth = 64 maxImportSources = 1024 maxGlobEntries = 1_000_000 ) type importLimits struct { depth int sources int bytes int64 nodes int entries int } func defaultImportLimits() importLimits { return importLimits{ depth: maxImportDepth, sources: maxImportSources, bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries, } } type importRoute struct { canonical string locations []Location key importNodeKey } type importAccounting struct { sources int bytes int64 nodes int entries int } type importTraversal struct { access *filesystemAccess accounting *importAccounting limits importLimits rule *VersionRule versionState *sourceVersionState defaults importDefaults scope acquisitionScope itemScope bool itemPrefix Path itemFallback Path sources []SourceInfo active map[resourceIdentity]int visited map[resourceIdentity]importRoute stack []importRoute importGraph importInclusionGraph } func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedDocumentsTracked(source, context, main, Path{}, rule, limits, &importAccounting{}, nil, nil) } func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedDocumentsTrackedWithVersion(source, context, main, destination, rule, limits, accounting, visited, hostToken, nil) } func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { var processing *sourceVersionProcessingResult if versionState != nil && versionState.mode != sourceVersionModeDefault { processing = versionState.mainProcessing } return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing, sequencePolicy{}) } func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedDocumentsTrackedWithProcessingDeferred(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing, policy, false) } func loadImportedDocumentsTrackedWithProcessingDeferred(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy, deferAssembly bool) (*composedDocument, []SourceInfo, []Diagnostic) { if nativeContext(source, context) { access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, deferAssembly) } rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } access, err := openHostAccess(rootPath) if err != nil { diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) diagnostic.Primary = nil return nil, sourceInfos(main.source), []Diagnostic{diagnostic} } if hostToken != nil { access.token = hostToken } return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, deferAssembly) } func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTracked(access, basePath, mainPath, main, Path{}, rule, limits, accounting, nil) } func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, nil) } func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { var processing *sourceVersionProcessingResult if versionState != nil && versionState.mode != sourceVersionModeDefault { processing = versionState.mainProcessing } return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTrackedAndProcessingPolicy(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, sequencePolicy{}) } func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) { return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, false) } func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy, deferAssembly bool) (*composedDocument, []SourceInfo, []Diagnostic) { if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} } for _, directory := range []string{access.rootPath, basePath} { info, err := access.stat(directory) if err != nil || !info.IsDir() { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)} } } mainInfo := cloneSourceInfo(*main.source) mainInfo.Destination = destination.clone() mainIsItem := main.source.placementMode == placementModeSequenceItem if mainPath != "" { mainInfo.Path = mainPath } mainResource := access.identity(mainPath) mainNodeKey := importKeyFor(access, mainPath, mainPath == "") if mainPath == "" { mainResource = access.syntheticIdentity() } if previous, repeated := visited[mainResource]; repeated { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})} } accounting.sources++ accounting.bytes += main.bytes accounting.nodes += countInputNodes(main.document) defaults := importDefaults{} if versionState != nil { defaults = snapshotImportDefaults(versionState.importDefaults) } state := importTraversal{ access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState, defaults: defaults, scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()}, sources: []SourceInfo{mainInfo}, active: make(map[resourceIdentity]int), visited: visited, importGraph: newImportInclusionGraph(), } if mainIsItem { state.itemScope = true state.itemPrefix = destination.clone() state.itemFallback = destination.clone() state.scope.destination = Path{} state.sources[0] = cloneSourceInfo(mainInfo) } if state.visited == nil { state.visited = make(map[resourceIdentity]importRoute) } if diagnostic := state.checkInitialLimits(); diagnostic != nil { _ = access.close() return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} } mainIdentity := mainPath if mainIdentity == "" { mainIdentity = "" } mainRoute := importRoute{canonical: mainIdentity} state.importGraph.addNode(importGraphNode{key: mainNodeKey, canonical: mainIdentity, sourceID: mainInfo.ID, role: importRoleMain}) mainRoute.key = mainNodeKey state.active[mainResource] = 0 state.visited[mainResource] = mainRoute state.stack = append(state.stack, mainRoute) var body *document.Node var declarations []importDeclarationSpec var diagnostics []Diagnostic if processing != nil { body = processing.payload declarations = processing.imports } else { body, declarations, diagnostics = splitImportDirectives(main.document) } if len(diagnostics) == 0 { if mainIsItem { state.scope.contributions = append(state.scope.contributions, acquiredContribution{ source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, payload: body, document: body, processing: processing, }) diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{}) } else { _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) placementNodes := destinationWrapperNodeCount(destination) if err != nil { diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} } else if accounting.nodes+placementNodes > limits.nodes { diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} } else { accounting.nodes += placementNodes state.scope.contributions = append(state.scope.contributions, acquiredContribution{ source: cloneSourceInfo(mainInfo), payload: body, document: placed, identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), processing: processing, }) diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) } } } if len(diagnostics) == 0 { state.scheduleAcquisitionScopes(&state.scope) } closeErr := access.close() if len(diagnostics) != 0 { if mainIsItem { diagnostics = prependCompositionDiagnosticPaths(diagnostics, destination.Index(0)) } return nil, cloneSourceInfos(state.sources), diagnostics } if closeErr != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} } acquisition := state.scope if mainIsItem { localScope := state.scope acquisition = acquisitionScope{ owner: mainNodeKey, destination: destination.clone(), contributions: []acquiredContribution{{ source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), processing: processing, pendingItem: &localScope, }}, } } if deferAssembly { if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} } return &composedDocument{ acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, }, cloneSourceInfos(state.sources), nil } acquired := &composedDocument{ acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, } input := assembledInput{ parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition), sources: state.sources, versionState: versionState, accounting: accounting, } composed, diagnostics := composeAcquiredInputWithPolicy(input, policy) return composed, cloneSourceInfos(state.sources), diagnostics } // composeAcquiredInputWithPolicy is the per-input adapter from retained // acquisition scopes to the established strict-composition behavior. It uses // the already acquired documents and graph; source handles are closed before // this handoff. func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { return composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy) } func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { return input.composition, nil } acquired := input.composition if len(acquired.acquisition) != 1 { return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)} } scope := cloneAcquisitionScope(acquired.acquisition[0]) var rule *VersionRule if input.versionState != nil { rule = input.versionState.effectiveRule } accounting := input.accounting if accounting == nil { accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes} } traversal := importTraversal{ accounting: accounting, limits: defaultImportLimits(), rule: rule, versionState: input.versionState, importGraph: acquired.importGraph.clone(), } if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil { main := scope.contributions[0] localScope := cloneAcquisitionScope(*main.pendingItem) if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 { return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) } documents := make([]*document.Node, 0, len(localScope.contributions)) for _, contribution := range localScope.contributions { documents = append(documents, contribution.document) } completed, diagnostics := composeDocuments(documents...) if len(diagnostics) != 0 { return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) } if input.versionState != nil { protectionRule := input.versionState.localRule if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil { prefix := main.destination.Index(0) relative, inside := relativeHistoryPath(prefix, protectionRule.Path) if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) { protectionRule = nil } else { scoped := *protectionRule scoped.Path = relative protectionRule = &scoped } } if diagnostic := importedVersionClaim(documents, localScope, completed, protectionRule); diagnostic != nil { if rule != nil { diagnostic.Path = rule.Path.clone() } return nil, []Diagnostic{*diagnostic} } } completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} projectNestedEvidence(completed, localScope) _, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} } if !main.placementCharged { placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem) if accounting.nodes+placementNodes > traversal.limits.nodes { return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)} } accounting.nodes += placementNodes } composition, diagnostics := composeDocumentsWithPolicy(policy, placed) if len(diagnostics) != 0 { return nil, diagnostics } synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{ source: cloneSourceInfo(main.source), payload: completed.root, document: placed, identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner, destination: main.destination.clone(), processing: main.processing, nested: completed, }}} projectNestedEvidence(composition, synthetic) projectAcquiredDestinations(composition, synthetic, input.sources) composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} composition.inputBytes = accounting.bytes composition.inputNodes = accounting.nodes composition.importGraph = acquired.importGraph.clone() return composition, nil } return composeAcquiredScopeWithPolicy( &scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting, ) } // composeAcquiredScopeWithPolicy is shared by per-input and underlay-group // handoffs. Lower policy is carried separately from registered application // constraints; current strict composition continues to use its existing rules. func composeAcquiredScopeWithPolicy( scope *acquisitionScope, lower collectionPolicyScope, policy sequencePolicy, graph importInclusionGraph, sources []SourceInfo, rule *VersionRule, versionState *sourceVersionState, accounting *importAccounting, ) (*composedDocument, []Diagnostic) { _ = lower traversal := importTraversal{ accounting: accounting, limits: defaultImportLimits(), rule: rule, versionState: versionState, importGraph: graph.clone(), } if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 { return nil, diagnostics } documents := make([]*document.Node, 0, len(scope.contributions)) for _, contribution := range scope.contributions { documents = append(documents, contribution.document) } composition, diagnostics := composeDocumentsWithPolicy(policy, documents...) if len(diagnostics) != 0 { return nil, diagnostics } if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil { return nil, []Diagnostic{*diagnostic} } projectNestedEvidence(composition, *scope) projectAcquiredDestinations(composition, *scope, sources) composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)} composition.inputBytes = accounting.bytes composition.inputNodes = accounting.nodes composition.importGraph = graph.clone() return composition, nil } func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic { if scope == nil { return nil } for index := range scope.contributions { contribution := &scope.contributions[index] if contribution.pendingItem == nil { continue } placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) if accounting.nodes+placementNodes > limits.nodes { diagnostic := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom) return &diagnostic } accounting.nodes += placementNodes contribution.placementCharged = true if diagnostic := reservePendingItemPlacements(contribution.pendingItem, limits, accounting); diagnostic != nil { return diagnostic } } return nil } func (s *importTraversal) checkInitialLimits() *Diagnostic { if s.accounting.sources > s.limits.sources { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) return &value } if s.accounting.bytes > s.limits.bytes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) return &value } if s.accounting.nodes > s.limits.nodes { value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) return &value } return nil } func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic { policies := make([]resolvedImportPolicy, len(declarations)) for index, spec := range declarations { policy, diagnostics := resolveImportPolicy(spec, s.defaults) if len(diagnostics) != 0 { return diagnostics } if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 { site := spec.fieldSites["placement"] return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}} } policies[index] = policy } for index, spec := range declarations { policy := policies[index] var selection importSelection var diagnostics []Diagnostic switch spec.selector { case importSelectorPath: selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy) case importSelectorDirectory: selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting) case importSelectorGlob: selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting) default: location := spec.location return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)} } if len(diagnostics) != 0 { return diagnostics } childDestination := appendDestination(parentDestination, policy.destination) // Count admitted records rather than lexical candidates: optional missing // paths are omitted during admission, and a partially present list is not empty. sourcesBefore := len(s.sources) for _, candidate := range selection.candidates { if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 { return diagnostics } } if !policy.required && len(s.sources) == sourcesBefore { s.recordEmptyImportSelection(spec) } } return nil } func (s *importTraversal) recordEmptyImportSelection(spec importDeclarationSpec) { for index := range s.sources { if s.sources[index].ID != spec.location.SourceID { continue } s.sources[index].importSelections = append(s.sources[index].importSelections, importSelectionEvidence{ path: spec.path.clone(), location: spec.location, selector: spec.selector, outcome: importSelectionNone, }) return } } func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic { location := candidate.site.location if location.SourceID == "" && location.Line == 0 && location.Column == 0 { location = spec.location } if !s.access.contains(candidate.accessPath) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} } canonical, err := s.access.canonical(candidate.accessPath) if err != nil { if !policy.required && errors.Is(err, fs.ErrNotExist) { return nil } code := CodeImportInvalid message := "import target could not be canonicalized" if errors.Is(err, fs.ErrNotExist) { code, message = CodeImportMissing, "required import target does not exist" } else if errors.Is(err, fs.ErrPermission) { message = "import target cannot be accessed" } return []Diagnostic{importDiagnostic(code, message, &location)} } if !s.access.contains(canonical) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} } if depth > s.limits.depth { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} } currentRoute := s.currentRoute(location) identity := s.access.identity(canonical) nodeKey := importKeyFor(s.access, canonical, false) importerKey := s.currentImportNode() if activeIndex, ok := s.active[identity]; ok { closingEdge := s.importGraph.addEdge(importerKey, nodeKey, location) if diagnostic, ok := s.importGraph.scheduleCycle(closingEdge, location); ok { return []Diagnostic{diagnostic} } cycle := append([]importRoute(nil), s.stack[activeIndex:]...) return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} } if previous, ok := s.visited[identity]; ok { s.importGraph.addEdge(importerKey, nodeKey, location) return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} } if s.accounting.sources+1 > s.limits.sources { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} } if _, err := s.access.relative(canonical); err != nil { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} } sourceID := "source-" + strconv.Itoa(s.accounting.sources+1) kind := SourceKindFile label := candidate.label if label == "" { label = candidate.path } if s.access.native != nil { kind = SourceKindFilesystem label = s.access.label + "/" + canonical } info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} if policy.placement == placementModeSequenceItem { info.placementMode = placementModeSequenceItem } // A source's own item placement still has an exact sequence-container // destination. Only an unresolved enclosing item index makes the path a // fallback container for this snapshot. info.destinationUnresolved = s.itemScope if kind == SourceKindFilesystem { info.namespace = s.access.token } s.sources = append(s.sources, info) data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required) if diagnostic != nil { return []Diagnostic{*diagnostic} } if skipped { s.sources = s.sources[:len(s.sources)-1] return nil } s.accounting.sources++ root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ Depth: maxDocumentDepth, Nodes: maxDocumentNodes, }) if len(issues) != 0 { return parserDiagnostics(sourceID, issues) } nodeCount := countInputNodes(root) if s.accounting.nodes+nodeCount > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } s.accounting.nodes += nodeCount var body *document.Node var declarations []importDeclarationSpec var diagnostics []Diagnostic var processing *sourceVersionProcessingResult if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault { versionDestination := destination if s.itemScope { versionDestination = appendDestination(s.itemPrefix.Index(0), destination) } processed, failures := s.versionState.process(root, sourceVersionContribution, versionDestination, sourceID, policy.placement) if len(failures) != 0 { return failures } processing = &processed body, declarations = processed.payload, processed.imports } else { body, declarations, diagnostics = splitImportDirectives(root) if len(diagnostics) != 0 { return diagnostics } } placementDestination := destination mode := placementModeMapping if policy.placement == placementModeSequenceItem { mode = placementModeSequenceItem } role := importRoleMapping if mode == placementModeSequenceItem { role = importRoleItem } s.importGraph.addNode(importGraphNode{key: nodeKey, canonical: canonical, sourceID: sourceID, role: role}) s.importGraph.addEdge(importerKey, nodeKey, location) if mode == placementModeSequenceItem { route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} s.active[identity] = len(s.stack) s.visited[identity] = route s.stack = append(s.stack, route) diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, processing) delete(s.active, identity) s.stack = s.stack[:len(s.stack)-1] return diagnostics } _, placed, err := placeDocument(body, placementDestination, body.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} } placementNodes := destinationWrapperNodeCount(placementDestination) if s.accounting.nodes+placementNodes > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } s.accounting.nodes += placementNodes route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} s.scope.contributions = append(s.scope.contributions, acquiredContribution{ source: cloneSourceInfo(info), payload: body, document: placed, identity: nodeKey, role: role, scopeOwner: s.scope.owner, destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, }) s.active[identity] = len(s.stack) s.visited[identity] = route s.stack = append(s.stack, route) diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth, destination) delete(s.active, identity) s.stack = s.stack[:len(s.stack)-1] return diagnostics } func (s *importTraversal) currentImportNode() importNodeKey { if len(s.stack) != 0 { return s.stack[len(s.stack)-1].key } key := s.scope.owner if _, exists := s.importGraph.index[key]; !exists { s.importGraph.addNode(importGraphNode{key: key, canonical: "", role: importRoleMain}) } return key } func (s *importTraversal) sourceDestination(destination Path) Path { if !s.itemScope { return destination.clone() } // Until the enclosing item receives its assigned index, only the collection // container is exact. Keep the local suffix in the acquisition projection. return s.itemFallback.clone() } func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic { parentScope := s.scope parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback s.scope = acquisitionScope{owner: identity} s.itemScope = true if parentItemScope { s.itemPrefix = appendDestination(parentPrefix, destination) } else { s.itemPrefix = destination.clone() s.itemFallback = destination.clone() } s.scope.contributions = append(s.scope.contributions, acquiredContribution{ source: cloneSourceInfo(info), payload: body, document: body, identity: identity, role: importRoleItem, scopeOwner: identity, processing: processing, }) diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{}) itemScope := s.scope s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback if len(diagnostics) != 0 { return diagnostics } itemScope.destination = Path{} s.scope.contributions = append(s.scope.contributions, acquiredContribution{ source: cloneSourceInfo(info), payload: body, identity: identity, role: importRoleItem, scopeOwner: parentScope.owner, destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, pendingItem: &itemScope, }) return nil } func (s *importTraversal) scheduleAcquisitionScopes(scope *acquisitionScope) { *scope = scheduleAcquisitionScope(*scope, s.importGraph) for index := range scope.contributions { if pending := scope.contributions[index].pendingItem; pending != nil { s.scheduleAcquisitionScopes(pending) } } } func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnostic { for index := range scope.contributions { contribution := &scope.contributions[index] if contribution.pendingItem == nil { continue } itemScope := contribution.pendingItem if diagnostics := s.composePendingItems(itemScope); len(diagnostics) != 0 { return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) } documents := make([]*document.Node, 0, len(itemScope.contributions)) for _, nested := range itemScope.contributions { documents = append(documents, nested.document) } completed, diagnostics := composeDocuments(documents...) if len(diagnostics) != 0 { return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) } // Version ownership is checked by the enclosing composition once this // item has its effective coordinates, not against its local field names. completed.acquisition = []acquisitionScope{cloneAcquisitionScope(*itemScope)} projectNestedEvidence(completed, *itemScope) _, placed, err := placeDocumentWithMode(completed.root, contribution.destination, placementModeSequenceItem, contribution.payload.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", contribution.source.ImportedFrom)} } placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) if !contribution.placementCharged { if s.accounting.nodes+placementNodes > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)} } s.accounting.nodes += placementNodes } contribution.payload = completed.root contribution.document = placed contribution.nested = completed contribution.pendingItem = nil } return nil } func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic { if composed == nil || rule == nil { return nil } mainIndex := -1 for index, contribution := range scope.contributions { if contribution.role == importRoleMain { mainIndex = index break } } ranges := indexSequenceRanges(composed.sequenceRanges) for index, contribution := range documents { if index == mainIndex { continue } if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil { location := publicLocation(node.Location) return &Diagnostic{ Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(), Message: "imported fragments cannot contribute to the schema-version path", Primary: &location, } } } return nil } func prependCompositionDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic { cloned := append([]Diagnostic(nil), diagnostics...) for index := range cloned { if !isCompositionDiagnostic(cloned[index]) { continue } cloned[index].Path = appendDestination(prefix, cloned[index].Path) cloned[index].Primary = cloneLocation(cloned[index].Primary) cloned[index].Related = cloneLocations(cloned[index].Related) } return cloned } func isCompositionDiagnostic(diagnostic Diagnostic) bool { return diagnostic.Code == CodeComposeConflict || diagnostic.Code == CodeComposeInvalid } // On a composition failure only the earlier contributions are needed to locate // this item. Project one enclosing scope at a time as traversal unwinds; syntax // and access diagnostics keep their original source-local coordinates. func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path, beforeContribution int) []Diagnostic { for _, diagnostic := range diagnostics { if isCompositionDiagnostic(diagnostic) { prefix := parent prefix.contributions = parent.contributions[:beforeContribution] index := priorSequenceLength([]acquisitionScope{prefix}, destination) return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index)) } } return diagnostics } // This scan is confined to failed loads, not the successful construction path. func priorSequenceLength(scopes []acquisitionScope, path Path) int { length := 0 for _, scope := range scopes { for _, contribution := range scope.contributions { if node, present := documentAtPath(contribution.document, path); present && node.Kind == document.Sequence { length += len(node.Sequence) } } } return length } func projectPriorInputDiagnosticPaths(diagnostics []Diagnostic, scopes []acquisitionScope) []Diagnostic { cloned := cloneDiagnostics(diagnostics) for index := range cloned { if !isCompositionDiagnostic(cloned[index]) { continue } for position, segment := range cloned[index].Path.segments { if segment.kind != pathSegmentIndex { continue } container := Path{segments: cloned[index].Path.segments[:position]} cloned[index].Path.segments[position].index += priorSequenceLength(scopes, container) // Registration does not address paths through item indexes. A prior // input shifts the outer collection, not its item-local sequences. break } } return cloned } func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { if composed == nil { return } applications := composed.applications projectedApplications := make([]historyApplication, 0, len(applications)) ranges := indexSequenceRanges(composed.sequenceRanges) for sourceIndex, contribution := range scope.contributions { if contribution.nested == nil { projectedApplications = append(projectedApplications, applications[sourceIndex]) continue } itemIndex := 0 if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { if projected, ok := assigned.outputForInput(0); ok { itemIndex = projected } } prefix := contribution.destination.Index(itemIndex) if sourceIndex < len(applications) { outer := applications[sourceIndex] structural := document.Clone(outer.value) if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > 0 { // The retained fragment is still a local singleton. Its assigned // outer index belongs to the projection, not this source tree. node.Sequence[0] = &document.Node{Kind: document.Mapping, Location: node.Sequence[0].Location} } outer.value = structural projectedApplications = append(projectedApplications, outer) for _, nested := range contribution.nested.applications { projected := nested projected.path = appendDestination(prefix, nested.path) projected.sequences = cloneSequenceHistoryContributions(nested.sequences) projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences) projectedApplications = append(projectedApplications, projected) } } projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix) } composed.applications = projectedApplications } func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) { var visit func(*document.Node, Path) visit = func(node *document.Node, path Path) { if node == nil { return } if origin, ok := origins[path.String()]; ok { target[appendDestination(prefix, path).String()] = cloneOrigin(origin) } for _, entry := range node.Mapping { visit(entry.Value, path.Key(entry.Key)) } for index, child := range node.Sequence { visit(child, path.Index(index)) } } visit(root, Path{}) } func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) { byID := make(map[string]int, len(sources)) for index := range sources { byID[sources[index].ID] = index } var projectScope func(*composedDocument, acquisitionScope, Path, bool, string) projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) { ranges := indexSequenceRanges(currentComposition.sequenceRanges) for sourceIndex, contribution := range current.contributions { if contribution.source.ID == parentSourceID { continue } contributionPrefix := prefix contributionInsideItem := insideItem itemIndex := 0 if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { if projected, ok := assigned.outputForInput(0); ok { itemIndex = projected } } if contribution.source.placementMode == placementModeSequenceItem { container := appendDestination(contributionPrefix, contribution.destination) contribution.source.Destination = container.clone() contribution.source.destinationUnresolved = false current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) if position, ok := byID[contribution.source.ID]; ok { sources[position].Destination = container sources[position].destinationUnresolved = false } contributionPrefix = appendDestination(container, Path{}.Index(itemIndex)) contributionInsideItem = true projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex) } else if contributionInsideItem { exactDestination := appendDestination(contributionPrefix, contribution.destination) contribution.source.Destination = exactDestination.clone() contribution.source.destinationUnresolved = false current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) if position, ok := byID[contribution.source.ID]; ok { sources[position].Destination = exactDestination sources[position].destinationUnresolved = false } projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex) } else { projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex) } if contribution.nested != nil && contribution.nested.acquisition != nil { for _, nestedScope := range contribution.nested.acquisition { projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID) } } } } projectScope(composed, scope, Path{}, false, "") } func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) { if processing != nil && processing.evidence != nil { evidence := processing.evidence evidence.destination = destination.clone() // Rebuild from source-local paths on each handoff: an input can be // composed first on its own and then at a different offset in its tier. evidence.projectedPaths = nil paths := evidence.sensitivityPaths() for index, path := range paths { for position := range path.segments { prefix := Path{segments: path.segments[:position]} assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}] if !found { continue } item, ok := schemaIndex(path.segments[position]) if ok && path.segments[position].kind != pathSegmentKey { if outputIndex, projected := assigned.outputForInput(item); projected { path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex} } } } paths[index] = path } evidence.projectedPaths = paths } } func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) { remaining := s.limits.bytes - s.accounting.bytes limit := min(int64(maxSourceBytes), remaining) if limit < 0 { limit = 0 } data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) if failure != nil { if !required && failure.missingTarget { return nil, true, nil } value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") return nil, false, &value } s.accounting.bytes += int64(len(data)) return data, false, nil } func (s *importTraversal) currentRoute(location Location) importRoute { locations := make([]Location, 0, len(s.stack)+1) for _, route := range s.stack { if len(route.locations) != 0 { locations = append(locations, route.locations[len(route.locations)-1]) } } locations = append(locations, location) return importRoute{locations: locations} } func sourceInfos(source *SourceInfo) []SourceInfo { if source == nil { return nil } return []SourceInfo{cloneSourceInfo(*source)} } func importDiagnostic(code Code, message string, location *Location) Diagnostic { return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)} } func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic { paths := make([]string, 0, len(cycle)+1) related := make([]Location, 0, len(cycle)) for _, route := range cycle { paths = append(paths, route.canonical) if len(route.locations) != 0 { related = append(related, route.locations[len(route.locations)-1]) } } paths = append(paths, target) return Diagnostic{ Code: CodeImportCycle, Severity: SeverityError, Message: "import cycle detected: " + strings.Join(paths, " -> "), Primary: &location, Related: related, } } func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic { related := append([]Location(nil), previous.locations...) if len(current.locations) > 1 { related = append(related, current.locations[:len(current.locations)-1]...) } return Diagnostic{ Code: CodeImportRepeated, Severity: SeverityError, Message: "configuration source is included by more than one import route", Primary: &location, Related: related, } }