From 6bc625b68f107b9ccb98d4558b4379e9c488a2ce Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 25 Oct 2026 13:21:29 +0000 Subject: [PATCH] Split import acquisition and projection responsibilities --- docs/roadmap/implementation.md | 10 +- internal/engine/import_acquisition.go | 501 ++++++++++++ internal/engine/import_projection.go | 277 +++++++ internal/engine/import_scope.go | 287 +++++++ internal/engine/imports.go | 1059 +------------------------ 5 files changed, 1074 insertions(+), 1060 deletions(-) create mode 100644 internal/engine/import_acquisition.go create mode 100644 internal/engine/import_projection.go create mode 100644 internal/engine/import_scope.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 548cb10..ac15dea 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Consolidation Implementation Plan -Status: in progress. Stages 1 through 9 are complete; follow Stage 10 through Stage 20 in order. +Status: in progress. Stages 1 through 10 are complete; follow Stage 11 through Stage 20 in order. The target executor is a Luna coding agent working through one end-to-end sprint. ## Authority, Baseline, And Execution @@ -453,7 +453,7 @@ validation timing, source evidence, budgets, version protection, and close reporting. Validation group A, consumer examples, the full Go suite, and `git diff --check` passed. -## Stage 10 — Give Import Responsibilities Focused Files +## Stage 10 — Give Import Responsibilities Focused Files (Complete) **Owners:** engine `imports.go` and new `import_acquisition.go`, `import_scope.go`, `import_projection.go`. @@ -476,6 +476,12 @@ reporting. Validation group A, consumer examples, the full Go suite, and **Finish:** maintainers can locate acquisition, local strict assembly, and evidence work separately. Keep this stage structural; parse/admission extraction follows. +Progress notes (2026-10-25): moved request acquisition and traversal admission to +`import_acquisition.go`, local strict composition and item-scope assembly to +`import_scope.go`, and origin, destination, version-evidence, and diagnostic-path +projection to `import_projection.go`. Shared limits, traversal data, and import +diagnostics remain in `imports.go`. `./scripts/check.sh` passed. + ## Stage 11 — Extract Import Declaration Parsing Operations **Owner:** engine `import_spec.go`. diff --git a/internal/engine/import_acquisition.go b/internal/engine/import_acquisition.go new file mode 100644 index 0000000..b16361d --- /dev/null +++ b/internal/engine/import_acquisition.go @@ -0,0 +1,501 @@ +package engine + +import ( + "bytes" + "errors" + "io/fs" + "strconv" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +func acquireImportedInput(request importAcquisitionRequest, policy *acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (*composedDocument, []SourceInfo, *resolvedFilesystemContext, []Diagnostic) { + source := request.input.input.source + context := request.input.input.filesystem + main := request.main + resolved, diagnostic := resolveFilesystemContext(source, context, main, tier.hostToken) + if diagnostic != nil { + return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic} + } + if resolved.native == nil { + if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil { + return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic} + } + } + access, err := openResolvedFilesystemAccess(resolved) + if err != nil { + diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) + diagnostic.Primary = nil + return nil, sourceInfos(main.source), nil, []Diagnostic{diagnostic} + } + composition, sources, diagnostics := acquireImportedWithAccess(request, policy, state, tier, access, resolved) + if len(diagnostics) != 0 { + return composition, sources, nil, diagnostics + } + return composition, sources, &resolved, nil +} + +func acquireImportedWithAccess(request importAcquisitionRequest, policy *acquisitionPolicy, loadState *acquisitionState, tier *tierAcquisitionState, access *filesystemAccess, resolved resolvedFilesystemContext) (*composedDocument, []SourceInfo, []Diagnostic) { + main := request.main + destination := request.input.input.destination + limits := policy.importLimits + accounting := loadState.accounting + versionState := loadState.versionState + processing := request.processing + basePath, mainPath := resolved.basePath, resolved.mainPath + if tier.visited == nil { + tier.visited = make(map[resourceIdentity]importRoute) + } + visited := tier.visited + if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { + _ = access.close() + return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} + } + for _, directory := range []string{access.rootPath, basePath} { + info, err := access.stat(directory) + if err != nil || !info.IsDir() { + _ = access.close() + return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)} + } + } + mainInfo := cloneSourceInfo(*main.source) + mainInfo.Destination = destination.clone() + mainIsItem := main.source.placementMode == placementModeSequenceItem + if mainPath != "" { + mainInfo.Path = mainPath + } + mainResource := access.identity(mainPath) + mainNodeKey := importKeyFor(access, mainPath, mainPath == "") + if mainPath == "" { + mainResource = access.syntheticIdentity() + } + if previous, repeated := visited[mainResource]; repeated { + _ = access.close() + return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})} + } + accounting.sources++ + accounting.bytes += main.bytes + accounting.nodes += countInputNodes(main.document) + directiveContext := request.input.directiveContext + if directiveContext == 0 { + directiveContext = rootDirectiveContextImportsOnly + } + if processing != nil && processing.directiveContext != 0 { + directiveContext = processing.directiveContext + } + request.input.directiveContext = directiveContext + state := importTraversal{ + access: access, request: request, policy: policy, state: loadState, tier: tier, + scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()}, + sources: []SourceInfo{mainInfo}, + active: make(map[resourceIdentity]int), + importGraph: newImportInclusionGraph(), + } + if mainIsItem { + state.itemScope = true + state.itemPrefix = destination.clone() + state.itemFallback = destination.clone() + state.scope.destination = Path{} + state.sources[0] = cloneSourceInfo(mainInfo) + } + + if diagnostic := state.checkInitialLimits(); diagnostic != nil { + _ = access.close() + return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} + } + + mainIdentity := mainPath + if mainIdentity == "" { + mainIdentity = "" + } + mainRoute := importRoute{canonical: mainIdentity} + state.importGraph.addNode(importGraphNode{key: mainNodeKey, canonical: mainIdentity, sourceID: mainInfo.ID, role: importRoleMain}) + mainRoute.key = mainNodeKey + state.active[mainResource] = 0 + tier.visited[mainResource] = mainRoute + state.stack = append(state.stack, mainRoute) + var body *document.Node + var declarations []importDeclarationSpec + var sequenceDeclarations []sequenceDeclarationSpec + var diagnostics []Diagnostic + if processing != nil { + body = processing.payload + declarations = processing.imports + sequenceDeclarations = cloneSequenceDeclarations(processing.sequences) + } else { + body, declarations, diagnostics = splitImportDirectives(main.document) + } + if len(diagnostics) == 0 { + if mainIsItem { + state.scope.contributions = append(state.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain, + scopeOwner: mainNodeKey, payload: body, document: body, processing: processing, + sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), + }) + diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{}) + } else { + _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) + placementNodes := destinationWrapperNodeCount(destination) + if err != nil { + diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} + } else if accounting.nodes+placementNodes > limits.nodes { + diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} + } else { + accounting.nodes += placementNodes + state.scope.contributions = append(state.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(mainInfo), payload: body, document: placed, + identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, + destination: destination.clone(), processing: processing, + sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), + }) + diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) + } + } + } + if len(diagnostics) == 0 { + state.scheduleAcquisitionScopes(&state.scope) + } + + closeErr := access.close() + if len(diagnostics) != 0 { + if mainIsItem { + diagnostics = prependCompositionDiagnosticPaths(diagnostics, destination.Index(0)) + } + return nil, cloneSourceInfos(state.sources), diagnostics + } + if closeErr != nil { + return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} + } + acquisition := state.scope + if mainIsItem { + localScope := state.scope + acquisition = acquisitionScope{ + owner: mainNodeKey, destination: destination.clone(), + contributions: []acquiredContribution{{ + source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey, + role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), + processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), pendingItem: &localScope, + }}, + } + } + if request.input.deferAssembly { + if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil { + return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} + } + return &composedDocument{ + acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, + importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, + }, cloneSourceInfos(state.sources), nil + } + acquired := &composedDocument{ + acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, + importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, + } + input := assembledInput{ + parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition), + sources: state.sources, versionState: versionState, accounting: accounting, + } + composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy.sequencePolicy) + return composed, cloneSourceInfos(state.sources), diagnostics +} + +func (s *importTraversal) checkInitialLimits() *Diagnostic { + if s.state.accounting.sources > s.policy.importLimits.sources { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) + return &value + } + if s.state.accounting.bytes > s.policy.importLimits.bytes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) + return &value + } + if s.state.accounting.nodes > s.policy.importLimits.nodes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) + return &value + } + return nil +} + +func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic { + policies := make([]resolvedImportPolicy, len(declarations)) + for index, spec := range declarations { + policy, diagnostics := resolveImportPolicy(spec, s.policy.importDefaults) + if len(diagnostics) != 0 { + return diagnostics + } + if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 { + site := spec.fieldSites["placement"] + return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}} + } + policies[index] = policy + } + for index, spec := range declarations { + policy := policies[index] + var selection importSelection + var diagnostics []Diagnostic + switch spec.selector { + case importSelectorPath: + selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy) + case importSelectorDirectory: + selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting) + case importSelectorGlob: + selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting) + default: + location := spec.location + return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)} + } + if len(diagnostics) != 0 { + return diagnostics + } + childDestination := appendDestination(parentDestination, policy.destination) + // Count admitted records rather than lexical candidates: optional missing + // paths are omitted during admission, and a partially present list is not empty. + sourcesBefore := len(s.sources) + for _, candidate := range selection.candidates { + if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 { + return diagnostics + } + } + if !policy.required && len(s.sources) == sourcesBefore { + s.recordEmptyImportSelection(spec) + } + } + return nil +} + +func (s *importTraversal) recordEmptyImportSelection(spec importDeclarationSpec) { + for index := range s.sources { + if s.sources[index].ID != spec.location.SourceID { + continue + } + s.sources[index].importSelections = append(s.sources[index].importSelections, importSelectionEvidence{ + path: spec.path.clone(), location: spec.location, + selector: spec.selector, outcome: importSelectionNone, + }) + return + } +} + +func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic { + location := candidate.site.location + if location.SourceID == "" && location.Line == 0 && location.Column == 0 { + location = spec.location + } + if !s.access.contains(candidate.accessPath) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} + } + canonical, err := s.access.canonical(candidate.accessPath) + if err != nil { + if !policy.required && errors.Is(err, fs.ErrNotExist) { + return nil + } + code := CodeImportInvalid + message := "import target could not be canonicalized" + if errors.Is(err, fs.ErrNotExist) { + code, message = CodeImportMissing, "required import target does not exist" + } else if errors.Is(err, fs.ErrPermission) { + message = "import target cannot be accessed" + } + return []Diagnostic{importDiagnostic(code, message, &location)} + } + if !s.access.contains(canonical) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} + } + if depth > s.policy.importLimits.depth { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} + } + + currentRoute := s.currentRoute(location) + identity := s.access.identity(canonical) + nodeKey := importKeyFor(s.access, canonical, false) + importerKey := s.currentImportNode() + if activeIndex, ok := s.active[identity]; ok { + closingEdge := s.importGraph.addEdge(importerKey, nodeKey, location) + if diagnostic, ok := s.importGraph.scheduleCycle(closingEdge, location); ok { + return []Diagnostic{diagnostic} + } + cycle := append([]importRoute(nil), s.stack[activeIndex:]...) + return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} + } + if previous, ok := s.tier.visited[identity]; ok { + s.importGraph.addEdge(importerKey, nodeKey, location) + return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} + } + if s.state.accounting.sources+1 > s.policy.importLimits.sources { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} + } + + if _, err := s.access.relative(canonical); err != nil { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} + } + sourceID := "source-" + strconv.Itoa(s.state.accounting.sources+1) + kind := SourceKindFile + label := candidate.label + if label == "" { + label = candidate.path + } + if s.access.native != nil { + kind = SourceKindFilesystem + label = s.access.label + "/" + canonical + } + info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} + if policy.placement == placementModeSequenceItem { + info.placementMode = placementModeSequenceItem + } + // A source's own item placement still has an exact sequence-container + // destination. Only an unresolved enclosing item index makes the path a + // fallback container for this snapshot. + info.destinationUnresolved = s.itemScope + if kind == SourceKindFilesystem { + info.namespace = s.access.token + } + s.sources = append(s.sources, info) + data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required) + if diagnostic != nil { + return []Diagnostic{*diagnostic} + } + if skipped { + s.sources = s.sources[:len(s.sources)-1] + return nil + } + s.state.accounting.sources++ + + root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ + Depth: maxDocumentDepth, Nodes: maxDocumentNodes, + }) + if len(issues) != 0 { + return parserDiagnostics(sourceID, issues) + } + nodeCount := countInputNodes(root) + if s.state.accounting.nodes+nodeCount > s.policy.importLimits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} + } + s.state.accounting.nodes += nodeCount + + var body *document.Node + var declarations []importDeclarationSpec + var sequenceDeclarations []sequenceDeclarationSpec + var diagnostics []Diagnostic + var processing *sourceVersionProcessingResult + directiveContext := s.request.input.directiveContext + if directiveContext == 0 { + directiveContext = rootDirectiveContextImportsOnly + } + if s.state.versionState != nil && s.state.versionState.mode != sourceVersionModeDefault { + versionDestination := destination + if s.itemScope { + versionDestination = appendDestination(s.itemPrefix.Index(0), destination) + } + processed, failures := s.state.versionState.processWithDirectiveContext(root, sourceVersionContribution, versionDestination, sourceID, policy.placement, directiveContext) + if len(failures) != 0 { + return failures + } + processing = &processed + body, declarations = processed.payload, processed.imports + } else { + versionDestination := destination + if s.itemScope { + versionDestination = appendDestination(s.itemPrefix.Index(0), destination) + } + processed := processSourceVersionWithDirectiveContext(root, s.request.input.protectionRule, sourceVersionContribution, versionDestination, sourceVersionModeDefault, directiveContext) + if len(processed.diagnostics) != 0 { + return processed.diagnostics + } + processing = &processed + body, declarations = processed.payload, processed.imports + } + sequenceDeclarations = cloneSequenceDeclarations(processing.sequences) + placementDestination := destination + mode := placementModeMapping + if policy.placement == placementModeSequenceItem { + mode = placementModeSequenceItem + } + role := importRoleMapping + if mode == placementModeSequenceItem { + role = importRoleItem + } + s.importGraph.addNode(importGraphNode{key: nodeKey, canonical: canonical, sourceID: sourceID, role: role}) + s.importGraph.addEdge(importerKey, nodeKey, location) + if mode == placementModeSequenceItem { + route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} + s.active[identity] = len(s.stack) + s.tier.visited[identity] = route + s.stack = append(s.stack, route) + diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, sequenceDeclarations, processing) + delete(s.active, identity) + s.stack = s.stack[:len(s.stack)-1] + return diagnostics + } + _, placed, err := placeDocument(body, placementDestination, body.Location, maxDocumentDepth, maxDocumentNodes) + if err != nil { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} + } + placementNodes := destinationWrapperNodeCount(placementDestination) + if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} + } + s.state.accounting.nodes += placementNodes + route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} + s.scope.contributions = append(s.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(info), payload: body, document: placed, + identity: nodeKey, role: role, scopeOwner: s.scope.owner, + destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, + sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), + }) + s.active[identity] = len(s.stack) + s.tier.visited[identity] = route + s.stack = append(s.stack, route) + diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth, destination) + delete(s.active, identity) + s.stack = s.stack[:len(s.stack)-1] + return diagnostics +} + +func (s *importTraversal) currentImportNode() importNodeKey { + if len(s.stack) != 0 { + return s.stack[len(s.stack)-1].key + } + key := s.scope.owner + if _, exists := s.importGraph.index[key]; !exists { + s.importGraph.addNode(importGraphNode{key: key, canonical: "", role: importRoleMain}) + } + return key +} + +func (s *importTraversal) sourceDestination(destination Path) Path { + if !s.itemScope { + return destination.clone() + } + // Until the enclosing item receives its assigned index, only the collection + // container is exact. Keep the local suffix in the acquisition projection. + return s.itemFallback.clone() +} + +func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) { + remaining := s.policy.importLimits.bytes - s.state.accounting.bytes + limit := min(int64(maxSourceBytes), remaining) + if limit < 0 { + limit = 0 + } + data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) + if failure != nil { + if !required && failure.missingTarget { + return nil, true, nil + } + value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") + return nil, false, &value + } + s.state.accounting.bytes += int64(len(data)) + return data, false, nil +} + +func (s *importTraversal) currentRoute(location Location) importRoute { + locations := make([]Location, 0, len(s.stack)+1) + for _, route := range s.stack { + if len(route.locations) != 0 { + locations = append(locations, route.locations[len(route.locations)-1]) + } + } + locations = append(locations, location) + return importRoute{locations: locations} +} diff --git a/internal/engine/import_projection.go b/internal/engine/import_projection.go new file mode 100644 index 0000000..69da1af --- /dev/null +++ b/internal/engine/import_projection.go @@ -0,0 +1,277 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +func prependCompositionDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic { + cloned := append([]Diagnostic(nil), diagnostics...) + for index := range cloned { + if !isCompositionDiagnostic(cloned[index]) { + continue + } + cloned[index].Path = appendDestination(prefix, cloned[index].Path) + cloned[index].Primary = cloneLocation(cloned[index].Primary) + cloned[index].Related = cloneLocations(cloned[index].Related) + } + return cloned +} + +func isCompositionDiagnostic(diagnostic Diagnostic) bool { + return diagnostic.Code == CodeComposeConflict || diagnostic.Code == CodeComposeInvalid +} + +// On a composition failure only the earlier contributions are needed to locate +// this item. Project one enclosing scope at a time as traversal unwinds; syntax +// and access diagnostics keep their original source-local coordinates. +func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path, beforeContribution int) []Diagnostic { + for _, diagnostic := range diagnostics { + if isCompositionDiagnostic(diagnostic) { + prefix := parent + prefix.contributions = parent.contributions[:beforeContribution] + index := priorSequenceLength([]acquisitionScope{prefix}, destination) + return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index)) + } + } + return diagnostics +} + +// This scan is confined to failed loads, not the successful construction path. +func priorSequenceLength(scopes []acquisitionScope, path Path) int { + length := 0 + for _, scope := range scopes { + for _, contribution := range scope.contributions { + if node, present := documentAtPath(contribution.document, path); present && node.Kind == document.Sequence { + length += len(node.Sequence) + } + } + } + return length +} + +func projectPriorInputDiagnosticPaths(diagnostics []Diagnostic, scopes []acquisitionScope) []Diagnostic { + cloned := cloneDiagnostics(diagnostics) + for index := range cloned { + if !isCompositionDiagnostic(cloned[index]) { + continue + } + for position, segment := range cloned[index].Path.segments { + if segment.kind != pathSegmentIndex { + continue + } + container := Path{segments: cloned[index].Path.segments[:position]} + cloned[index].Path.segments[position].index += priorSequenceLength(scopes, container) + // Registration does not address paths through item indexes. A prior + // input shifts the outer collection, not its item-local sequences. + break + } + } + return cloned +} + +func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { + if composed == nil { + return + } + applications := composed.applications + projectedApplications := make([]historyApplication, 0, len(applications)) + ranges := indexSequenceRanges(composed.sequenceRanges) + for sourceIndex, contribution := range scope.contributions { + if contribution.nested == nil { + projectedApplications = append(projectedApplications, applications[sourceIndex]) + continue + } + itemIndex := 0 + if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { + if projected, ok := assigned.outputForInput(0); ok { + itemIndex = projected + } + } + prefix := contribution.destination.Index(itemIndex) + if sourceIndex < len(applications) { + outer := applications[sourceIndex] + structural := document.Clone(outer.value) + if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > 0 { + // The retained fragment is still a local singleton. Its assigned + // outer index belongs to the projection, not this source tree. + node.Sequence[0] = &document.Node{Kind: document.Mapping, Location: node.Sequence[0].Location} + } + outer.value = structural + projectedApplications = append(projectedApplications, outer) + for _, nested := range contribution.nested.applications { + projected := nested + projected.path = appendDestination(prefix, nested.path) + projected.sequences = cloneSequenceHistoryContributions(nested.sequences) + projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences) + projectedApplications = append(projectedApplications, projected) + } + } + projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix) + } + composed.applications = projectedApplications + projectNestedCollectionPolicies(composed, scope, ranges) +} + +func projectNestedCollectionPolicies(composed *composedDocument, scope acquisitionScope, ranges map[sequenceRangeKey]sequenceRange) { + if composed == nil { + return + } + type itemPolicyProjection struct { + destination Path + policy collectionPolicy + base int + } + projected := make(map[string]*itemPolicyProjection) + var prefixLengths map[string]int + for sourceIndex, contribution := range scope.contributions { + if prefixLengths != nil && sourceIndex > 0 { + addSequencePrefixLengths(scope.contributions[sourceIndex-1].document, Path{}, prefixLengths) + } + if contribution.nested == nil || len(contribution.nested.collectionPolicies.collections) == 0 { + continue + } + key := contribution.destination.String() + assigned := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: key}] + itemIndex, found := assigned.outputForInput(0) + if !found { + // Normal composition supplies the assignment directly. For a + // handoff without ranges, scan each preceding tree only once. + if prefixLengths == nil { + prefixLengths = make(map[string]int) + for _, previous := range scope.contributions[:sourceIndex] { + addSequencePrefixLengths(previous.document, Path{}, prefixLengths) + } + } + itemIndex = prefixLengths[key] + } + working, initialized := projected[key] + if !initialized { + // Read one owned copy per destination, then add each incoming + // record without recopying earlier records and their scopes. + policy, _ := composed.collectionPolicies.at(contribution.destination) + working = &itemPolicyProjection{destination: contribution.destination, policy: policy, base: len(policy.recordOrder)} + projected[key] = working + } + working.policy.setRecordPolicyAt(working.base+itemIndex, collectionRecordPolicy{ + owner: newCollectionRecordPolicy().owner, + scope: contribution.nested.collectionPolicies, + }) + } + for _, working := range projected { + composed.collectionPolicies.set(working.destination, working.policy) + } +} + +func addSequencePrefixLengths(node *document.Node, path Path, lengths map[string]int) { + if node == nil { + return + } + if node.Kind == document.Sequence { + lengths[path.String()] += len(node.Sequence) + return + } + // Contribution destinations select mapping paths, not sequence members. + for _, entry := range node.Mapping { + addSequencePrefixLengths(entry.Value, path.Key(entry.Key), lengths) + } +} + +func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) { + var visit func(*document.Node, Path) + visit = func(node *document.Node, path Path) { + if node == nil { + return + } + if origin, ok := origins[path.String()]; ok { + target[appendDestination(prefix, path).String()] = cloneOrigin(origin) + } + for _, entry := range node.Mapping { + visit(entry.Value, path.Key(entry.Key)) + } + for index, child := range node.Sequence { + visit(child, path.Index(index)) + } + } + visit(root, Path{}) +} + +func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) { + byID := make(map[string]int, len(sources)) + for index := range sources { + byID[sources[index].ID] = index + } + var projectScope func(*composedDocument, acquisitionScope, Path, bool, string) + projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) { + ranges := indexSequenceRanges(currentComposition.sequenceRanges) + for sourceIndex, contribution := range current.contributions { + if contribution.source.ID == parentSourceID { + continue + } + contributionPrefix := prefix + contributionInsideItem := insideItem + itemIndex := 0 + if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { + if projected, ok := assigned.outputForInput(0); ok { + itemIndex = projected + } + } + if contribution.source.placementMode == placementModeSequenceItem { + container := appendDestination(contributionPrefix, contribution.destination) + contribution.source.Destination = container.clone() + contribution.source.destinationUnresolved = false + current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) + if position, ok := byID[contribution.source.ID]; ok { + sources[position].Destination = container + sources[position].destinationUnresolved = false + } + contributionPrefix = appendDestination(container, Path{}.Index(itemIndex)) + contributionInsideItem = true + projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex) + } else if contributionInsideItem { + exactDestination := appendDestination(contributionPrefix, contribution.destination) + contribution.source.Destination = exactDestination.clone() + contribution.source.destinationUnresolved = false + current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) + if position, ok := byID[contribution.source.ID]; ok { + sources[position].Destination = exactDestination + sources[position].destinationUnresolved = false + } + projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex) + } else { + projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex) + } + if contribution.nested != nil && contribution.nested.acquisition != nil { + for _, nestedScope := range contribution.nested.acquisition { + projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID) + } + } + } + } + projectScope(composed, scope, Path{}, false, "") +} + +func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) { + if processing != nil && processing.evidence != nil { + evidence := processing.evidence + evidence.destination = destination.clone() + // Rebuild from source-local paths on each handoff: an input can be + // composed first on its own and then at a different offset in its tier. + evidence.projectedPaths = nil + paths := evidence.sensitivityPaths() + for index, path := range paths { + for position := range path.segments { + prefix := Path{segments: path.segments[:position]} + assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}] + if !found { + continue + } + item, ok := schemaIndex(path.segments[position]) + if ok && path.segments[position].kind != pathSegmentKey { + if outputIndex, projected := assigned.outputForInput(item); projected { + path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex} + } + } + } + paths[index] = path + } + evidence.projectedPaths = paths + } +} diff --git a/internal/engine/import_scope.go b/internal/engine/import_scope.go new file mode 100644 index 0000000..9d3e4ba --- /dev/null +++ b/internal/engine/import_scope.go @@ -0,0 +1,287 @@ +package engine + +import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + +func compositionTraversal(policy sequencePolicy, rule *VersionRule, versionState *sourceVersionState, accounting *importAccounting, graph importInclusionGraph) importTraversal { + return importTraversal{ + request: importAcquisitionRequest{input: inputAcquisitionRequest{protectionRule: rule}}, + policy: &acquisitionPolicy{sequencePolicy: policy, importLimits: defaultImportLimits()}, + state: &acquisitionState{accounting: accounting, versionState: versionState}, + tier: &tierAcquisitionState{}, + importGraph: graph.clone(), + } +} + +// composeAcquiredInputWithPolicy is the per-input adapter from retained +// acquisition scopes to the established strict-composition behavior. It uses +// the already acquired documents and graph; source handles are closed before +// this handoff. +func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { + if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { + return input.composition, nil + } + acquired := input.composition + if len(acquired.acquisition) != 1 { + return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)} + } + scope := cloneAcquisitionScope(acquired.acquisition[0]) + var rule *VersionRule + if input.versionState != nil { + rule = input.versionState.effectiveRule + } + accounting := input.accounting + if accounting == nil { + accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes} + } + traversal := compositionTraversal(policy, rule, input.versionState, accounting, acquired.importGraph) + if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil { + main := scope.contributions[0] + localScope := cloneAcquisitionScope(*main.pendingItem) + if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 { + return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) + } + documents := make([]*document.Node, 0, len(localScope.contributions)) + for _, contribution := range localScope.contributions { + documents = append(documents, contribution.document) + } + completed, diagnostics := composeDocumentsWithCollectionPolicy( + policy, localScope.collectionPolicies, + acquiredSequenceDeclarations(localScope), documents..., + ) + if len(diagnostics) != 0 { + return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) + } + if input.versionState != nil { + protectionRule := input.versionState.localRule + if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil { + prefix := main.destination.Index(0) + relative, inside := relativeHistoryPath(prefix, protectionRule.Path) + if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) { + protectionRule = nil + } else { + scoped := *protectionRule + scoped.Path = relative + protectionRule = &scoped + } + } + if diagnostic := importedVersionClaim(documents, localScope, completed, protectionRule); diagnostic != nil { + if rule != nil { + diagnostic.Path = rule.Path.clone() + } + return nil, []Diagnostic{*diagnostic} + } + } + completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} + projectNestedEvidence(completed, localScope) + _, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes) + if err != nil { + return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} + } + if !main.placementCharged { + placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem) + if accounting.nodes+placementNodes > traversal.policy.importLimits.nodes { + return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)} + } + accounting.nodes += placementNodes + } + composition, diagnostics := composeDocumentsWithPolicy(policy, placed) + if len(diagnostics) != 0 { + return nil, diagnostics + } + synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{ + source: cloneSourceInfo(main.source), payload: completed.root, document: placed, + identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner, + destination: main.destination.clone(), processing: main.processing, nested: completed, + }}} + projectNestedEvidence(composition, synthetic) + projectAcquiredDestinations(composition, synthetic, input.sources) + composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} + composition.inputBytes = accounting.bytes + composition.inputNodes = accounting.nodes + composition.importGraph = acquired.importGraph.clone() + return composition, nil + } + return composeAcquiredScopeWithPolicy( + &scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting, + ) +} + +// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group +// handoffs. Lower policy is carried separately from registered application +// constraints; current strict composition continues to use its existing rules. +func composeAcquiredScopeWithPolicy( + scope *acquisitionScope, + lower collectionPolicyScope, + policy sequencePolicy, + graph importInclusionGraph, + sources []SourceInfo, + rule *VersionRule, + versionState *sourceVersionState, + accounting *importAccounting, +) (*composedDocument, []Diagnostic) { + traversal := compositionTraversal(policy, rule, versionState, accounting, graph) + if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 { + return nil, diagnostics + } + documents := make([]*document.Node, 0, len(scope.contributions)) + for _, contribution := range scope.contributions { + documents = append(documents, contribution.document) + } + composition, diagnostics := composeDocumentsWithCollectionPolicy( + policy, lower, acquiredSequenceDeclarations(*scope), documents..., + ) + if len(diagnostics) != 0 { + return nil, diagnostics + } + if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil { + return nil, []Diagnostic{*diagnostic} + } + projectNestedEvidence(composition, *scope) + projectAcquiredDestinations(composition, *scope, sources) + composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)} + composition.inputBytes = accounting.bytes + composition.inputNodes = accounting.nodes + composition.importGraph = graph.clone() + return composition, nil +} + +func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic { + if scope == nil { + return nil + } + for index := range scope.contributions { + contribution := &scope.contributions[index] + if contribution.pendingItem == nil { + continue + } + placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) + if accounting.nodes+placementNodes > limits.nodes { + diagnostic := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom) + return &diagnostic + } + accounting.nodes += placementNodes + contribution.placementCharged = true + if diagnostic := reservePendingItemPlacements(contribution.pendingItem, limits, accounting); diagnostic != nil { + return diagnostic + } + } + return nil +} + +func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, sequenceDeclarations []sequenceDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic { + parentScope := s.scope + parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback + s.scope = acquisitionScope{owner: identity} + s.itemScope = true + if parentItemScope { + s.itemPrefix = appendDestination(parentPrefix, destination) + } else { + s.itemPrefix = destination.clone() + s.itemFallback = destination.clone() + } + s.scope.contributions = append(s.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(info), payload: body, document: body, + identity: identity, role: importRoleItem, scopeOwner: identity, + processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), + }) + diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{}) + itemScope := s.scope + s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback + if len(diagnostics) != 0 { + return diagnostics + } + itemScope.destination = Path{} + s.scope.contributions = append(s.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(info), payload: body, + identity: identity, role: importRoleItem, scopeOwner: parentScope.owner, + destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, + pendingItem: &itemScope, + }) + return nil +} + +func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnostic { + for index := range scope.contributions { + contribution := &scope.contributions[index] + if contribution.pendingItem == nil { + continue + } + itemScope := contribution.pendingItem + if diagnostics := s.composePendingItems(itemScope); len(diagnostics) != 0 { + return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) + } + documents := make([]*document.Node, 0, len(itemScope.contributions)) + for _, nested := range itemScope.contributions { + documents = append(documents, nested.document) + } + // An item scope is below a sequence index. Application selectors match + // mapping-only effective paths, so none can apply inside this scope. + // Its YAML declarations remain relative to the owning record. + completed, diagnostics := composeDocumentsWithCollectionPolicy( + sequencePolicy{}, itemScope.collectionPolicies, + acquiredSequenceDeclarations(*itemScope), documents..., + ) + if len(diagnostics) != 0 { + return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) + } + // Version ownership is checked by the enclosing composition once this + // item has its effective coordinates, not against its local field names. + completed.acquisition = []acquisitionScope{cloneAcquisitionScope(*itemScope)} + projectNestedEvidence(completed, *itemScope) + _, placed, err := placeDocumentWithMode(completed.root, contribution.destination, placementModeSequenceItem, contribution.payload.Location, maxDocumentDepth, maxDocumentNodes) + if err != nil { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", contribution.source.ImportedFrom)} + } + placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) + if !contribution.placementCharged { + if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)} + } + s.state.accounting.nodes += placementNodes + } + contribution.payload = completed.root + contribution.document = placed + contribution.nested = completed + contribution.pendingItem = nil + } + return nil +} + +func acquiredSequenceDeclarations(scope acquisitionScope) []sequenceDeclarationSpec { + var declarations []sequenceDeclarationSpec + for _, contribution := range scope.contributions { + projected := cloneSequenceDeclarations(contribution.sequenceDeclarations) + for index := range projected { + projected[index].destination = appendDestination(contribution.relativeDestination, projected[index].destination) + } + declarations = append(declarations, projected...) + } + return declarations +} + +func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic { + if composed == nil || rule == nil { + return nil + } + mainIndex := -1 + for index, contribution := range scope.contributions { + if contribution.role == importRoleMain { + mainIndex = index + break + } + } + ranges := indexSequenceRanges(composed.sequenceRanges) + for index, contribution := range documents { + if index == mainIndex { + continue + } + if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil { + location := publicLocation(node.Location) + return &Diagnostic{ + Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(), + Message: "imported fragments cannot contribute to the schema-version path", Primary: &location, + } + } + } + return nil +} diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 14899e8..bc10df2 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -1,15 +1,6 @@ package engine -import ( - "bytes" - "errors" - "io/fs" - "strconv" - "strings" - - "gitea.maximumdirect.net/eric/ConfigKit/internal/document" - yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" -) +import "strings" const ( maxImportDepth = 64 @@ -62,665 +53,6 @@ type importTraversal struct { importGraph importInclusionGraph } -func compositionTraversal(policy sequencePolicy, rule *VersionRule, versionState *sourceVersionState, accounting *importAccounting, graph importInclusionGraph) importTraversal { - return importTraversal{ - request: importAcquisitionRequest{input: inputAcquisitionRequest{protectionRule: rule}}, - policy: &acquisitionPolicy{sequencePolicy: policy, importLimits: defaultImportLimits()}, - state: &acquisitionState{accounting: accounting, versionState: versionState}, - tier: &tierAcquisitionState{}, - importGraph: graph.clone(), - } -} - -func acquireImportedInput(request importAcquisitionRequest, policy *acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (*composedDocument, []SourceInfo, *resolvedFilesystemContext, []Diagnostic) { - source := request.input.input.source - context := request.input.input.filesystem - main := request.main - resolved, diagnostic := resolveFilesystemContext(source, context, main, tier.hostToken) - if diagnostic != nil { - return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic} - } - if resolved.native == nil { - if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil { - return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic} - } - } - access, err := openResolvedFilesystemAccess(resolved) - if err != nil { - diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{}) - diagnostic.Primary = nil - return nil, sourceInfos(main.source), nil, []Diagnostic{diagnostic} - } - composition, sources, diagnostics := acquireImportedWithAccess(request, policy, state, tier, access, resolved) - if len(diagnostics) != 0 { - return composition, sources, nil, diagnostics - } - return composition, sources, &resolved, nil -} - -func acquireImportedWithAccess(request importAcquisitionRequest, policy *acquisitionPolicy, loadState *acquisitionState, tier *tierAcquisitionState, access *filesystemAccess, resolved resolvedFilesystemContext) (*composedDocument, []SourceInfo, []Diagnostic) { - main := request.main - destination := request.input.input.destination - limits := policy.importLimits - accounting := loadState.accounting - versionState := loadState.versionState - processing := request.processing - basePath, mainPath := resolved.basePath, resolved.mainPath - if tier.visited == nil { - tier.visited = make(map[resourceIdentity]importRoute) - } - visited := tier.visited - if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { - _ = access.close() - return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} - } - for _, directory := range []string{access.rootPath, basePath} { - info, err := access.stat(directory) - if err != nil || !info.IsDir() { - _ = access.close() - return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)} - } - } - mainInfo := cloneSourceInfo(*main.source) - mainInfo.Destination = destination.clone() - mainIsItem := main.source.placementMode == placementModeSequenceItem - if mainPath != "" { - mainInfo.Path = mainPath - } - mainResource := access.identity(mainPath) - mainNodeKey := importKeyFor(access, mainPath, mainPath == "") - if mainPath == "" { - mainResource = access.syntheticIdentity() - } - if previous, repeated := visited[mainResource]; repeated { - _ = access.close() - return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})} - } - accounting.sources++ - accounting.bytes += main.bytes - accounting.nodes += countInputNodes(main.document) - directiveContext := request.input.directiveContext - if directiveContext == 0 { - directiveContext = rootDirectiveContextImportsOnly - } - if processing != nil && processing.directiveContext != 0 { - directiveContext = processing.directiveContext - } - request.input.directiveContext = directiveContext - state := importTraversal{ - access: access, request: request, policy: policy, state: loadState, tier: tier, - scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()}, - sources: []SourceInfo{mainInfo}, - active: make(map[resourceIdentity]int), - importGraph: newImportInclusionGraph(), - } - if mainIsItem { - state.itemScope = true - state.itemPrefix = destination.clone() - state.itemFallback = destination.clone() - state.scope.destination = Path{} - state.sources[0] = cloneSourceInfo(mainInfo) - } - - if diagnostic := state.checkInitialLimits(); diagnostic != nil { - _ = access.close() - return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} - } - - mainIdentity := mainPath - if mainIdentity == "" { - mainIdentity = "" - } - mainRoute := importRoute{canonical: mainIdentity} - state.importGraph.addNode(importGraphNode{key: mainNodeKey, canonical: mainIdentity, sourceID: mainInfo.ID, role: importRoleMain}) - mainRoute.key = mainNodeKey - state.active[mainResource] = 0 - tier.visited[mainResource] = mainRoute - state.stack = append(state.stack, mainRoute) - var body *document.Node - var declarations []importDeclarationSpec - var sequenceDeclarations []sequenceDeclarationSpec - var diagnostics []Diagnostic - if processing != nil { - body = processing.payload - declarations = processing.imports - sequenceDeclarations = cloneSequenceDeclarations(processing.sequences) - } else { - body, declarations, diagnostics = splitImportDirectives(main.document) - } - if len(diagnostics) == 0 { - if mainIsItem { - state.scope.contributions = append(state.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain, - scopeOwner: mainNodeKey, payload: body, document: body, processing: processing, - sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), - }) - diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{}) - } else { - _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) - placementNodes := destinationWrapperNodeCount(destination) - if err != nil { - diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} - } else if accounting.nodes+placementNodes > limits.nodes { - diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} - } else { - accounting.nodes += placementNodes - state.scope.contributions = append(state.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(mainInfo), payload: body, document: placed, - identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, - destination: destination.clone(), processing: processing, - sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), - }) - diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) - } - } - } - if len(diagnostics) == 0 { - state.scheduleAcquisitionScopes(&state.scope) - } - - closeErr := access.close() - if len(diagnostics) != 0 { - if mainIsItem { - diagnostics = prependCompositionDiagnosticPaths(diagnostics, destination.Index(0)) - } - return nil, cloneSourceInfos(state.sources), diagnostics - } - if closeErr != nil { - return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} - } - acquisition := state.scope - if mainIsItem { - localScope := state.scope - acquisition = acquisitionScope{ - owner: mainNodeKey, destination: destination.clone(), - contributions: []acquiredContribution{{ - source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey, - role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), - processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), pendingItem: &localScope, - }}, - } - } - if request.input.deferAssembly { - if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil { - return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} - } - return &composedDocument{ - acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, - importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, - }, cloneSourceInfos(state.sources), nil - } - acquired := &composedDocument{ - acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, - importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, - } - input := assembledInput{ - parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition), - sources: state.sources, versionState: versionState, accounting: accounting, - } - composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy.sequencePolicy) - return composed, cloneSourceInfos(state.sources), diagnostics -} - -// composeAcquiredInputWithPolicy is the per-input adapter from retained -// acquisition scopes to the established strict-composition behavior. It uses -// the already acquired documents and graph; source handles are closed before -// this handoff. -func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { - if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { - return input.composition, nil - } - acquired := input.composition - if len(acquired.acquisition) != 1 { - return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)} - } - scope := cloneAcquisitionScope(acquired.acquisition[0]) - var rule *VersionRule - if input.versionState != nil { - rule = input.versionState.effectiveRule - } - accounting := input.accounting - if accounting == nil { - accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes} - } - traversal := compositionTraversal(policy, rule, input.versionState, accounting, acquired.importGraph) - if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil { - main := scope.contributions[0] - localScope := cloneAcquisitionScope(*main.pendingItem) - if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 { - return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) - } - documents := make([]*document.Node, 0, len(localScope.contributions)) - for _, contribution := range localScope.contributions { - documents = append(documents, contribution.document) - } - completed, diagnostics := composeDocumentsWithCollectionPolicy( - policy, localScope.collectionPolicies, - acquiredSequenceDeclarations(localScope), documents..., - ) - if len(diagnostics) != 0 { - return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) - } - if input.versionState != nil { - protectionRule := input.versionState.localRule - if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil { - prefix := main.destination.Index(0) - relative, inside := relativeHistoryPath(prefix, protectionRule.Path) - if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) { - protectionRule = nil - } else { - scoped := *protectionRule - scoped.Path = relative - protectionRule = &scoped - } - } - if diagnostic := importedVersionClaim(documents, localScope, completed, protectionRule); diagnostic != nil { - if rule != nil { - diagnostic.Path = rule.Path.clone() - } - return nil, []Diagnostic{*diagnostic} - } - } - completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} - projectNestedEvidence(completed, localScope) - _, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes) - if err != nil { - return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} - } - if !main.placementCharged { - placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem) - if accounting.nodes+placementNodes > traversal.policy.importLimits.nodes { - return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)} - } - accounting.nodes += placementNodes - } - composition, diagnostics := composeDocumentsWithPolicy(policy, placed) - if len(diagnostics) != 0 { - return nil, diagnostics - } - synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{ - source: cloneSourceInfo(main.source), payload: completed.root, document: placed, - identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner, - destination: main.destination.clone(), processing: main.processing, nested: completed, - }}} - projectNestedEvidence(composition, synthetic) - projectAcquiredDestinations(composition, synthetic, input.sources) - composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} - composition.inputBytes = accounting.bytes - composition.inputNodes = accounting.nodes - composition.importGraph = acquired.importGraph.clone() - return composition, nil - } - return composeAcquiredScopeWithPolicy( - &scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting, - ) -} - -// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group -// handoffs. Lower policy is carried separately from registered application -// constraints; current strict composition continues to use its existing rules. -func composeAcquiredScopeWithPolicy( - scope *acquisitionScope, - lower collectionPolicyScope, - policy sequencePolicy, - graph importInclusionGraph, - sources []SourceInfo, - rule *VersionRule, - versionState *sourceVersionState, - accounting *importAccounting, -) (*composedDocument, []Diagnostic) { - traversal := compositionTraversal(policy, rule, versionState, accounting, graph) - if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 { - return nil, diagnostics - } - documents := make([]*document.Node, 0, len(scope.contributions)) - for _, contribution := range scope.contributions { - documents = append(documents, contribution.document) - } - composition, diagnostics := composeDocumentsWithCollectionPolicy( - policy, lower, acquiredSequenceDeclarations(*scope), documents..., - ) - if len(diagnostics) != 0 { - return nil, diagnostics - } - if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil { - return nil, []Diagnostic{*diagnostic} - } - projectNestedEvidence(composition, *scope) - projectAcquiredDestinations(composition, *scope, sources) - composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)} - composition.inputBytes = accounting.bytes - composition.inputNodes = accounting.nodes - composition.importGraph = graph.clone() - return composition, nil -} - -func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic { - if scope == nil { - return nil - } - for index := range scope.contributions { - contribution := &scope.contributions[index] - if contribution.pendingItem == nil { - continue - } - placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) - if accounting.nodes+placementNodes > limits.nodes { - diagnostic := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom) - return &diagnostic - } - accounting.nodes += placementNodes - contribution.placementCharged = true - if diagnostic := reservePendingItemPlacements(contribution.pendingItem, limits, accounting); diagnostic != nil { - return diagnostic - } - } - return nil -} - -func (s *importTraversal) checkInitialLimits() *Diagnostic { - if s.state.accounting.sources > s.policy.importLimits.sources { - value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) - return &value - } - if s.state.accounting.bytes > s.policy.importLimits.bytes { - value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) - return &value - } - if s.state.accounting.nodes > s.policy.importLimits.nodes { - value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) - return &value - } - return nil -} - -func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic { - policies := make([]resolvedImportPolicy, len(declarations)) - for index, spec := range declarations { - policy, diagnostics := resolveImportPolicy(spec, s.policy.importDefaults) - if len(diagnostics) != 0 { - return diagnostics - } - if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 { - site := spec.fieldSites["placement"] - return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}} - } - policies[index] = policy - } - for index, spec := range declarations { - policy := policies[index] - var selection importSelection - var diagnostics []Diagnostic - switch spec.selector { - case importSelectorPath: - selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy) - case importSelectorDirectory: - selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting) - case importSelectorGlob: - selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting) - default: - location := spec.location - return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)} - } - if len(diagnostics) != 0 { - return diagnostics - } - childDestination := appendDestination(parentDestination, policy.destination) - // Count admitted records rather than lexical candidates: optional missing - // paths are omitted during admission, and a partially present list is not empty. - sourcesBefore := len(s.sources) - for _, candidate := range selection.candidates { - if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 { - return diagnostics - } - } - if !policy.required && len(s.sources) == sourcesBefore { - s.recordEmptyImportSelection(spec) - } - } - return nil -} - -func (s *importTraversal) recordEmptyImportSelection(spec importDeclarationSpec) { - for index := range s.sources { - if s.sources[index].ID != spec.location.SourceID { - continue - } - s.sources[index].importSelections = append(s.sources[index].importSelections, importSelectionEvidence{ - path: spec.path.clone(), location: spec.location, - selector: spec.selector, outcome: importSelectionNone, - }) - return - } -} - -func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic { - location := candidate.site.location - if location.SourceID == "" && location.Line == 0 && location.Column == 0 { - location = spec.location - } - if !s.access.contains(candidate.accessPath) { - return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} - } - canonical, err := s.access.canonical(candidate.accessPath) - if err != nil { - if !policy.required && errors.Is(err, fs.ErrNotExist) { - return nil - } - code := CodeImportInvalid - message := "import target could not be canonicalized" - if errors.Is(err, fs.ErrNotExist) { - code, message = CodeImportMissing, "required import target does not exist" - } else if errors.Is(err, fs.ErrPermission) { - message = "import target cannot be accessed" - } - return []Diagnostic{importDiagnostic(code, message, &location)} - } - if !s.access.contains(canonical) { - return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} - } - if depth > s.policy.importLimits.depth { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} - } - - currentRoute := s.currentRoute(location) - identity := s.access.identity(canonical) - nodeKey := importKeyFor(s.access, canonical, false) - importerKey := s.currentImportNode() - if activeIndex, ok := s.active[identity]; ok { - closingEdge := s.importGraph.addEdge(importerKey, nodeKey, location) - if diagnostic, ok := s.importGraph.scheduleCycle(closingEdge, location); ok { - return []Diagnostic{diagnostic} - } - cycle := append([]importRoute(nil), s.stack[activeIndex:]...) - return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} - } - if previous, ok := s.tier.visited[identity]; ok { - s.importGraph.addEdge(importerKey, nodeKey, location) - return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} - } - if s.state.accounting.sources+1 > s.policy.importLimits.sources { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} - } - - if _, err := s.access.relative(canonical); err != nil { - return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} - } - sourceID := "source-" + strconv.Itoa(s.state.accounting.sources+1) - kind := SourceKindFile - label := candidate.label - if label == "" { - label = candidate.path - } - if s.access.native != nil { - kind = SourceKindFilesystem - label = s.access.label + "/" + canonical - } - info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} - if policy.placement == placementModeSequenceItem { - info.placementMode = placementModeSequenceItem - } - // A source's own item placement still has an exact sequence-container - // destination. Only an unresolved enclosing item index makes the path a - // fallback container for this snapshot. - info.destinationUnresolved = s.itemScope - if kind == SourceKindFilesystem { - info.namespace = s.access.token - } - s.sources = append(s.sources, info) - data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required) - if diagnostic != nil { - return []Diagnostic{*diagnostic} - } - if skipped { - s.sources = s.sources[:len(s.sources)-1] - return nil - } - s.state.accounting.sources++ - - root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ - Depth: maxDocumentDepth, Nodes: maxDocumentNodes, - }) - if len(issues) != 0 { - return parserDiagnostics(sourceID, issues) - } - nodeCount := countInputNodes(root) - if s.state.accounting.nodes+nodeCount > s.policy.importLimits.nodes { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} - } - s.state.accounting.nodes += nodeCount - - var body *document.Node - var declarations []importDeclarationSpec - var sequenceDeclarations []sequenceDeclarationSpec - var diagnostics []Diagnostic - var processing *sourceVersionProcessingResult - directiveContext := s.request.input.directiveContext - if directiveContext == 0 { - directiveContext = rootDirectiveContextImportsOnly - } - if s.state.versionState != nil && s.state.versionState.mode != sourceVersionModeDefault { - versionDestination := destination - if s.itemScope { - versionDestination = appendDestination(s.itemPrefix.Index(0), destination) - } - processed, failures := s.state.versionState.processWithDirectiveContext(root, sourceVersionContribution, versionDestination, sourceID, policy.placement, directiveContext) - if len(failures) != 0 { - return failures - } - processing = &processed - body, declarations = processed.payload, processed.imports - } else { - versionDestination := destination - if s.itemScope { - versionDestination = appendDestination(s.itemPrefix.Index(0), destination) - } - processed := processSourceVersionWithDirectiveContext(root, s.request.input.protectionRule, sourceVersionContribution, versionDestination, sourceVersionModeDefault, directiveContext) - if len(processed.diagnostics) != 0 { - return processed.diagnostics - } - processing = &processed - body, declarations = processed.payload, processed.imports - } - sequenceDeclarations = cloneSequenceDeclarations(processing.sequences) - placementDestination := destination - mode := placementModeMapping - if policy.placement == placementModeSequenceItem { - mode = placementModeSequenceItem - } - role := importRoleMapping - if mode == placementModeSequenceItem { - role = importRoleItem - } - s.importGraph.addNode(importGraphNode{key: nodeKey, canonical: canonical, sourceID: sourceID, role: role}) - s.importGraph.addEdge(importerKey, nodeKey, location) - if mode == placementModeSequenceItem { - route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} - s.active[identity] = len(s.stack) - s.tier.visited[identity] = route - s.stack = append(s.stack, route) - diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, sequenceDeclarations, processing) - delete(s.active, identity) - s.stack = s.stack[:len(s.stack)-1] - return diagnostics - } - _, placed, err := placeDocument(body, placementDestination, body.Location, maxDocumentDepth, maxDocumentNodes) - if err != nil { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} - } - placementNodes := destinationWrapperNodeCount(placementDestination) - if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} - } - s.state.accounting.nodes += placementNodes - route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey} - s.scope.contributions = append(s.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(info), payload: body, document: placed, - identity: nodeKey, role: role, scopeOwner: s.scope.owner, - destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, - sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), - }) - s.active[identity] = len(s.stack) - s.tier.visited[identity] = route - s.stack = append(s.stack, route) - diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth, destination) - delete(s.active, identity) - s.stack = s.stack[:len(s.stack)-1] - return diagnostics -} - -func (s *importTraversal) currentImportNode() importNodeKey { - if len(s.stack) != 0 { - return s.stack[len(s.stack)-1].key - } - key := s.scope.owner - if _, exists := s.importGraph.index[key]; !exists { - s.importGraph.addNode(importGraphNode{key: key, canonical: "", role: importRoleMain}) - } - return key -} - -func (s *importTraversal) sourceDestination(destination Path) Path { - if !s.itemScope { - return destination.clone() - } - // Until the enclosing item receives its assigned index, only the collection - // container is exact. Keep the local suffix in the acquisition projection. - return s.itemFallback.clone() -} - -func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, sequenceDeclarations []sequenceDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic { - parentScope := s.scope - parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback - s.scope = acquisitionScope{owner: identity} - s.itemScope = true - if parentItemScope { - s.itemPrefix = appendDestination(parentPrefix, destination) - } else { - s.itemPrefix = destination.clone() - s.itemFallback = destination.clone() - } - s.scope.contributions = append(s.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(info), payload: body, document: body, - identity: identity, role: importRoleItem, scopeOwner: identity, - processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), - }) - diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{}) - itemScope := s.scope - s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback - if len(diagnostics) != 0 { - return diagnostics - } - itemScope.destination = Path{} - s.scope.contributions = append(s.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(info), payload: body, - identity: identity, role: importRoleItem, scopeOwner: parentScope.owner, - destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, - pendingItem: &itemScope, - }) - return nil -} - func (s *importTraversal) scheduleAcquisitionScopes(scope *acquisitionScope) { *scope = scheduleAcquisitionScope(*scope, s.importGraph) for index := range scope.contributions { @@ -730,395 +62,6 @@ func (s *importTraversal) scheduleAcquisitionScopes(scope *acquisitionScope) { } } -func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnostic { - for index := range scope.contributions { - contribution := &scope.contributions[index] - if contribution.pendingItem == nil { - continue - } - itemScope := contribution.pendingItem - if diagnostics := s.composePendingItems(itemScope); len(diagnostics) != 0 { - return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) - } - documents := make([]*document.Node, 0, len(itemScope.contributions)) - for _, nested := range itemScope.contributions { - documents = append(documents, nested.document) - } - // An item scope is below a sequence index. Application selectors match - // mapping-only effective paths, so none can apply inside this scope. - // Its YAML declarations remain relative to the owning record. - completed, diagnostics := composeDocumentsWithCollectionPolicy( - sequencePolicy{}, itemScope.collectionPolicies, - acquiredSequenceDeclarations(*itemScope), documents..., - ) - if len(diagnostics) != 0 { - return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index) - } - // Version ownership is checked by the enclosing composition once this - // item has its effective coordinates, not against its local field names. - completed.acquisition = []acquisitionScope{cloneAcquisitionScope(*itemScope)} - projectNestedEvidence(completed, *itemScope) - _, placed, err := placeDocumentWithMode(completed.root, contribution.destination, placementModeSequenceItem, contribution.payload.Location, maxDocumentDepth, maxDocumentNodes) - if err != nil { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", contribution.source.ImportedFrom)} - } - placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem) - if !contribution.placementCharged { - if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)} - } - s.state.accounting.nodes += placementNodes - } - contribution.payload = completed.root - contribution.document = placed - contribution.nested = completed - contribution.pendingItem = nil - } - return nil -} - -func acquiredSequenceDeclarations(scope acquisitionScope) []sequenceDeclarationSpec { - var declarations []sequenceDeclarationSpec - for _, contribution := range scope.contributions { - projected := cloneSequenceDeclarations(contribution.sequenceDeclarations) - for index := range projected { - projected[index].destination = appendDestination(contribution.relativeDestination, projected[index].destination) - } - declarations = append(declarations, projected...) - } - return declarations -} - -func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic { - if composed == nil || rule == nil { - return nil - } - mainIndex := -1 - for index, contribution := range scope.contributions { - if contribution.role == importRoleMain { - mainIndex = index - break - } - } - ranges := indexSequenceRanges(composed.sequenceRanges) - for index, contribution := range documents { - if index == mainIndex { - continue - } - if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil { - location := publicLocation(node.Location) - return &Diagnostic{ - Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(), - Message: "imported fragments cannot contribute to the schema-version path", Primary: &location, - } - } - } - return nil -} - -func prependCompositionDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic { - cloned := append([]Diagnostic(nil), diagnostics...) - for index := range cloned { - if !isCompositionDiagnostic(cloned[index]) { - continue - } - cloned[index].Path = appendDestination(prefix, cloned[index].Path) - cloned[index].Primary = cloneLocation(cloned[index].Primary) - cloned[index].Related = cloneLocations(cloned[index].Related) - } - return cloned -} - -func isCompositionDiagnostic(diagnostic Diagnostic) bool { - return diagnostic.Code == CodeComposeConflict || diagnostic.Code == CodeComposeInvalid -} - -// On a composition failure only the earlier contributions are needed to locate -// this item. Project one enclosing scope at a time as traversal unwinds; syntax -// and access diagnostics keep their original source-local coordinates. -func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path, beforeContribution int) []Diagnostic { - for _, diagnostic := range diagnostics { - if isCompositionDiagnostic(diagnostic) { - prefix := parent - prefix.contributions = parent.contributions[:beforeContribution] - index := priorSequenceLength([]acquisitionScope{prefix}, destination) - return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index)) - } - } - return diagnostics -} - -// This scan is confined to failed loads, not the successful construction path. -func priorSequenceLength(scopes []acquisitionScope, path Path) int { - length := 0 - for _, scope := range scopes { - for _, contribution := range scope.contributions { - if node, present := documentAtPath(contribution.document, path); present && node.Kind == document.Sequence { - length += len(node.Sequence) - } - } - } - return length -} - -func projectPriorInputDiagnosticPaths(diagnostics []Diagnostic, scopes []acquisitionScope) []Diagnostic { - cloned := cloneDiagnostics(diagnostics) - for index := range cloned { - if !isCompositionDiagnostic(cloned[index]) { - continue - } - for position, segment := range cloned[index].Path.segments { - if segment.kind != pathSegmentIndex { - continue - } - container := Path{segments: cloned[index].Path.segments[:position]} - cloned[index].Path.segments[position].index += priorSequenceLength(scopes, container) - // Registration does not address paths through item indexes. A prior - // input shifts the outer collection, not its item-local sequences. - break - } - } - return cloned -} - -func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { - if composed == nil { - return - } - applications := composed.applications - projectedApplications := make([]historyApplication, 0, len(applications)) - ranges := indexSequenceRanges(composed.sequenceRanges) - for sourceIndex, contribution := range scope.contributions { - if contribution.nested == nil { - projectedApplications = append(projectedApplications, applications[sourceIndex]) - continue - } - itemIndex := 0 - if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { - if projected, ok := assigned.outputForInput(0); ok { - itemIndex = projected - } - } - prefix := contribution.destination.Index(itemIndex) - if sourceIndex < len(applications) { - outer := applications[sourceIndex] - structural := document.Clone(outer.value) - if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > 0 { - // The retained fragment is still a local singleton. Its assigned - // outer index belongs to the projection, not this source tree. - node.Sequence[0] = &document.Node{Kind: document.Mapping, Location: node.Sequence[0].Location} - } - outer.value = structural - projectedApplications = append(projectedApplications, outer) - for _, nested := range contribution.nested.applications { - projected := nested - projected.path = appendDestination(prefix, nested.path) - projected.sequences = cloneSequenceHistoryContributions(nested.sequences) - projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences) - projectedApplications = append(projectedApplications, projected) - } - } - projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix) - } - composed.applications = projectedApplications - projectNestedCollectionPolicies(composed, scope, ranges) -} - -func projectNestedCollectionPolicies(composed *composedDocument, scope acquisitionScope, ranges map[sequenceRangeKey]sequenceRange) { - if composed == nil { - return - } - type itemPolicyProjection struct { - destination Path - policy collectionPolicy - base int - } - projected := make(map[string]*itemPolicyProjection) - var prefixLengths map[string]int - for sourceIndex, contribution := range scope.contributions { - if prefixLengths != nil && sourceIndex > 0 { - addSequencePrefixLengths(scope.contributions[sourceIndex-1].document, Path{}, prefixLengths) - } - if contribution.nested == nil || len(contribution.nested.collectionPolicies.collections) == 0 { - continue - } - key := contribution.destination.String() - assigned := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: key}] - itemIndex, found := assigned.outputForInput(0) - if !found { - // Normal composition supplies the assignment directly. For a - // handoff without ranges, scan each preceding tree only once. - if prefixLengths == nil { - prefixLengths = make(map[string]int) - for _, previous := range scope.contributions[:sourceIndex] { - addSequencePrefixLengths(previous.document, Path{}, prefixLengths) - } - } - itemIndex = prefixLengths[key] - } - working, initialized := projected[key] - if !initialized { - // Read one owned copy per destination, then add each incoming - // record without recopying earlier records and their scopes. - policy, _ := composed.collectionPolicies.at(contribution.destination) - working = &itemPolicyProjection{destination: contribution.destination, policy: policy, base: len(policy.recordOrder)} - projected[key] = working - } - working.policy.setRecordPolicyAt(working.base+itemIndex, collectionRecordPolicy{ - owner: newCollectionRecordPolicy().owner, - scope: contribution.nested.collectionPolicies, - }) - } - for _, working := range projected { - composed.collectionPolicies.set(working.destination, working.policy) - } -} - -func addSequencePrefixLengths(node *document.Node, path Path, lengths map[string]int) { - if node == nil { - return - } - if node.Kind == document.Sequence { - lengths[path.String()] += len(node.Sequence) - return - } - // Contribution destinations select mapping paths, not sequence members. - for _, entry := range node.Mapping { - addSequencePrefixLengths(entry.Value, path.Key(entry.Key), lengths) - } -} - -func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) { - var visit func(*document.Node, Path) - visit = func(node *document.Node, path Path) { - if node == nil { - return - } - if origin, ok := origins[path.String()]; ok { - target[appendDestination(prefix, path).String()] = cloneOrigin(origin) - } - for _, entry := range node.Mapping { - visit(entry.Value, path.Key(entry.Key)) - } - for index, child := range node.Sequence { - visit(child, path.Index(index)) - } - } - visit(root, Path{}) -} - -func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) { - byID := make(map[string]int, len(sources)) - for index := range sources { - byID[sources[index].ID] = index - } - var projectScope func(*composedDocument, acquisitionScope, Path, bool, string) - projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) { - ranges := indexSequenceRanges(currentComposition.sequenceRanges) - for sourceIndex, contribution := range current.contributions { - if contribution.source.ID == parentSourceID { - continue - } - contributionPrefix := prefix - contributionInsideItem := insideItem - itemIndex := 0 - if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found { - if projected, ok := assigned.outputForInput(0); ok { - itemIndex = projected - } - } - if contribution.source.placementMode == placementModeSequenceItem { - container := appendDestination(contributionPrefix, contribution.destination) - contribution.source.Destination = container.clone() - contribution.source.destinationUnresolved = false - current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) - if position, ok := byID[contribution.source.ID]; ok { - sources[position].Destination = container - sources[position].destinationUnresolved = false - } - contributionPrefix = appendDestination(container, Path{}.Index(itemIndex)) - contributionInsideItem = true - projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex) - } else if contributionInsideItem { - exactDestination := appendDestination(contributionPrefix, contribution.destination) - contribution.source.Destination = exactDestination.clone() - contribution.source.destinationUnresolved = false - current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) - if position, ok := byID[contribution.source.ID]; ok { - sources[position].Destination = exactDestination - sources[position].destinationUnresolved = false - } - projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex) - } else { - projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex) - } - if contribution.nested != nil && contribution.nested.acquisition != nil { - for _, nestedScope := range contribution.nested.acquisition { - projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID) - } - } - } - } - projectScope(composed, scope, Path{}, false, "") -} - -func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) { - if processing != nil && processing.evidence != nil { - evidence := processing.evidence - evidence.destination = destination.clone() - // Rebuild from source-local paths on each handoff: an input can be - // composed first on its own and then at a different offset in its tier. - evidence.projectedPaths = nil - paths := evidence.sensitivityPaths() - for index, path := range paths { - for position := range path.segments { - prefix := Path{segments: path.segments[:position]} - assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}] - if !found { - continue - } - item, ok := schemaIndex(path.segments[position]) - if ok && path.segments[position].kind != pathSegmentKey { - if outputIndex, projected := assigned.outputForInput(item); projected { - path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex} - } - } - } - paths[index] = path - } - evidence.projectedPaths = paths - } -} - -func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) { - remaining := s.policy.importLimits.bytes - s.state.accounting.bytes - limit := min(int64(maxSourceBytes), remaining) - if limit < 0 { - limit = 0 - } - data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) - if failure != nil { - if !required && failure.missingTarget { - return nil, true, nil - } - value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") - return nil, false, &value - } - s.state.accounting.bytes += int64(len(data)) - return data, false, nil -} - -func (s *importTraversal) currentRoute(location Location) importRoute { - locations := make([]Location, 0, len(s.stack)+1) - for _, route := range s.stack { - if len(route.locations) != 0 { - locations = append(locations, route.locations[len(route.locations)-1]) - } - } - locations = append(locations, location) - return importRoute{locations: locations} -} - func sourceInfos(source *SourceInfo) []SourceInfo { if source == nil { return nil