378 lines
13 KiB
Go
378 lines
13 KiB
Go
package engine
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io/fs"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
|
|
yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml"
|
|
)
|
|
|
|
const (
|
|
maxImportDepth = 64
|
|
maxImportSources = 1024
|
|
maxGlobEntries = 1_000_000
|
|
)
|
|
|
|
type importLimits struct {
|
|
depth int
|
|
sources int
|
|
bytes int64
|
|
nodes int
|
|
entries int
|
|
}
|
|
|
|
func defaultImportLimits() importLimits {
|
|
return importLimits{
|
|
depth: maxImportDepth, sources: maxImportSources,
|
|
bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries,
|
|
}
|
|
}
|
|
|
|
type importRoute struct {
|
|
canonical string
|
|
locations []Location
|
|
}
|
|
|
|
type importAccounting struct {
|
|
sources int
|
|
bytes int64
|
|
nodes int
|
|
entries int
|
|
}
|
|
|
|
type importTraversal struct {
|
|
access *filesystemAccess
|
|
accounting *importAccounting
|
|
limits importLimits
|
|
rule *VersionRule
|
|
|
|
documents []*document.Node
|
|
sources []SourceInfo
|
|
active map[resourceIdentity]int
|
|
visited map[resourceIdentity]importRoute
|
|
stack []importRoute
|
|
}
|
|
|
|
func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
|
|
return loadImportedDocumentsTracked(source, context, main, rule, limits, &importAccounting{}, nil, nil)
|
|
}
|
|
|
|
func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) {
|
|
if nativeContext(source, context) {
|
|
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
|
|
if diagnostic != nil {
|
|
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
|
|
}
|
|
return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, visited)
|
|
}
|
|
rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
|
|
if diagnostic != nil {
|
|
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
|
|
}
|
|
access, err := openHostAccess(rootPath)
|
|
if err != nil {
|
|
diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
|
|
diagnostic.Primary = nil
|
|
return nil, sourceInfos(main.source), []Diagnostic{diagnostic}
|
|
}
|
|
if hostToken != nil {
|
|
access.token = hostToken
|
|
}
|
|
return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, visited)
|
|
}
|
|
|
|
func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
|
|
return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, nil)
|
|
}
|
|
|
|
func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) {
|
|
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
|
|
_ = access.close()
|
|
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
|
|
}
|
|
for _, directory := range []string{access.rootPath, basePath} {
|
|
info, err := access.stat(directory)
|
|
if err != nil || !info.IsDir() {
|
|
_ = access.close()
|
|
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)}
|
|
}
|
|
}
|
|
mainInfo := cloneSourceInfo(*main.source)
|
|
if mainPath != "" {
|
|
mainInfo.Path = mainPath
|
|
}
|
|
mainKey := access.identity(mainPath)
|
|
if mainPath == "" {
|
|
mainKey = access.syntheticIdentity()
|
|
}
|
|
if previous, repeated := visited[mainKey]; repeated {
|
|
_ = access.close()
|
|
return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})}
|
|
}
|
|
accounting.sources++
|
|
accounting.bytes += main.bytes
|
|
accounting.nodes += countInputNodes(main.document)
|
|
state := importTraversal{
|
|
access: access, accounting: accounting, limits: limits, rule: rule,
|
|
documents: make([]*document.Node, 0, 1),
|
|
sources: []SourceInfo{mainInfo},
|
|
active: make(map[resourceIdentity]int), visited: visited,
|
|
}
|
|
if state.visited == nil {
|
|
state.visited = make(map[resourceIdentity]importRoute)
|
|
}
|
|
if diagnostic := state.checkInitialLimits(); diagnostic != nil {
|
|
_ = access.close()
|
|
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
|
|
}
|
|
|
|
mainIdentity := mainPath
|
|
if mainIdentity == "" {
|
|
mainIdentity = "<selected-source>"
|
|
}
|
|
mainRoute := importRoute{canonical: mainIdentity}
|
|
state.active[mainKey] = 0
|
|
state.visited[mainKey] = mainRoute
|
|
state.stack = append(state.stack, mainRoute)
|
|
body, declarations, diagnostics := splitImportDirectives(main.document)
|
|
if len(diagnostics) == 0 {
|
|
state.documents = append(state.documents, body)
|
|
diagnostics = state.walkDeclarations(basePath, declarations, 0)
|
|
}
|
|
|
|
closeErr := access.close()
|
|
if len(diagnostics) != 0 {
|
|
return nil, cloneSourceInfos(state.sources), diagnostics
|
|
}
|
|
if closeErr != nil {
|
|
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
|
|
}
|
|
composed, diagnostics := composeDocuments(state.documents...)
|
|
if composed != nil {
|
|
composed.inputBytes = accounting.bytes
|
|
composed.inputNodes = accounting.nodes
|
|
}
|
|
return composed, cloneSourceInfos(state.sources), diagnostics
|
|
}
|
|
|
|
func (s *importTraversal) checkInitialLimits() *Diagnostic {
|
|
if s.accounting.sources > s.limits.sources {
|
|
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil)
|
|
return &value
|
|
}
|
|
if s.accounting.bytes > s.limits.bytes {
|
|
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil)
|
|
return &value
|
|
}
|
|
if s.accounting.nodes > s.limits.nodes {
|
|
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)
|
|
return &value
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int) []Diagnostic {
|
|
for _, declaration := range declarations {
|
|
if declaration.glob != "" {
|
|
matches, diagnostics := s.expandImportGlob(basePath, declaration)
|
|
if len(diagnostics) != 0 {
|
|
return diagnostics
|
|
}
|
|
if len(matches) == 0 {
|
|
if declaration.optional {
|
|
continue
|
|
}
|
|
location := declaration.location
|
|
return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)}
|
|
}
|
|
for _, match := range matches {
|
|
matched := declaration
|
|
matched.path = match
|
|
matched.glob = ""
|
|
if diagnostics := s.visitPath(basePath, matched, depth+1); len(diagnostics) != 0 {
|
|
return diagnostics
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if diagnostics := s.visitPath(basePath, declaration, depth+1); len(diagnostics) != 0 {
|
|
return diagnostics
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, depth int) []Diagnostic {
|
|
location := declaration.location
|
|
if depth > s.limits.depth {
|
|
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
|
|
}
|
|
if !validImportPath(declaration.path) {
|
|
return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)}
|
|
}
|
|
|
|
candidate := s.access.join(basePath, declaration.path)
|
|
if !s.access.contains(candidate) {
|
|
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)}
|
|
}
|
|
canonical, err := s.access.canonical(candidate)
|
|
if err != nil {
|
|
if declaration.optional && errors.Is(err, fs.ErrNotExist) {
|
|
return nil
|
|
}
|
|
code := CodeImportInvalid
|
|
message := "import target could not be canonicalized"
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
code, message = CodeImportMissing, "required import target does not exist"
|
|
} else if errors.Is(err, fs.ErrPermission) {
|
|
message = "import target cannot be accessed"
|
|
}
|
|
return []Diagnostic{importDiagnostic(code, message, &location)}
|
|
}
|
|
if !s.access.contains(canonical) {
|
|
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)}
|
|
}
|
|
|
|
currentRoute := s.currentRoute(location)
|
|
identity := s.access.identity(canonical)
|
|
if activeIndex, ok := s.active[identity]; ok {
|
|
cycle := append([]importRoute(nil), s.stack[activeIndex:]...)
|
|
return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)}
|
|
}
|
|
if previous, ok := s.visited[identity]; ok {
|
|
return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)}
|
|
}
|
|
if s.accounting.sources+1 > s.limits.sources {
|
|
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)}
|
|
}
|
|
|
|
if _, err := s.access.relative(canonical); err != nil {
|
|
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)}
|
|
}
|
|
sourceID := "source-" + strconv.Itoa(s.accounting.sources+1)
|
|
kind := SourceKindFile
|
|
label := declaration.path
|
|
if s.access.native != nil {
|
|
kind = SourceKindFilesystem
|
|
label = s.access.label + "/" + canonical
|
|
}
|
|
info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Layer: LayerBase, Kind: kind, ImportedFrom: &location}
|
|
s.sources = append(s.sources, info)
|
|
data, skipped, diagnostic := s.readImport(canonical, sourceID, declaration.optional)
|
|
if diagnostic != nil {
|
|
return []Diagnostic{*diagnostic}
|
|
}
|
|
if skipped {
|
|
s.sources = s.sources[:len(s.sources)-1]
|
|
return nil
|
|
}
|
|
s.accounting.sources++
|
|
|
|
root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{
|
|
Depth: maxDocumentDepth, Nodes: maxDocumentNodes,
|
|
})
|
|
if len(issues) != 0 {
|
|
return parserDiagnostics(sourceID, issues)
|
|
}
|
|
nodeCount := countInputNodes(root)
|
|
if s.accounting.nodes+nodeCount > s.limits.nodes {
|
|
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
|
|
}
|
|
s.accounting.nodes += nodeCount
|
|
|
|
body, declarations, diagnostics := splitImportDirectives(root)
|
|
if len(diagnostics) != 0 {
|
|
return diagnostics
|
|
}
|
|
if versionNode := fragmentVersionClaim(body, s.rule); versionNode != nil {
|
|
versionLocation := publicLocation(versionNode.Location)
|
|
return []Diagnostic{{
|
|
Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(),
|
|
Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
|
|
}}
|
|
}
|
|
|
|
route := importRoute{canonical: canonical, locations: currentRoute.locations}
|
|
s.documents = append(s.documents, body)
|
|
s.active[identity] = len(s.stack)
|
|
s.visited[identity] = route
|
|
s.stack = append(s.stack, route)
|
|
diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth)
|
|
delete(s.active, identity)
|
|
s.stack = s.stack[:len(s.stack)-1]
|
|
return diagnostics
|
|
}
|
|
|
|
func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) {
|
|
remaining := s.limits.bytes - s.accounting.bytes
|
|
limit := min(int64(maxSourceBytes), remaining)
|
|
if limit < 0 {
|
|
limit = 0
|
|
}
|
|
data, failure := readRegularFile(name, s.access.stat, s.access.open, limit)
|
|
if failure != nil {
|
|
if optional && failure.missingTarget {
|
|
return nil, true, nil
|
|
}
|
|
value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit")
|
|
return nil, false, &value
|
|
}
|
|
s.accounting.bytes += int64(len(data))
|
|
return data, false, nil
|
|
}
|
|
|
|
func (s *importTraversal) currentRoute(location Location) importRoute {
|
|
locations := make([]Location, 0, len(s.stack)+1)
|
|
for _, route := range s.stack {
|
|
if len(route.locations) != 0 {
|
|
locations = append(locations, route.locations[len(route.locations)-1])
|
|
}
|
|
}
|
|
locations = append(locations, location)
|
|
return importRoute{locations: locations}
|
|
}
|
|
|
|
func sourceInfos(source *SourceInfo) []SourceInfo {
|
|
if source == nil {
|
|
return nil
|
|
}
|
|
return []SourceInfo{cloneSourceInfo(*source)}
|
|
}
|
|
|
|
func importDiagnostic(code Code, message string, location *Location) Diagnostic {
|
|
return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)}
|
|
}
|
|
|
|
func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic {
|
|
paths := make([]string, 0, len(cycle)+1)
|
|
related := make([]Location, 0, len(cycle))
|
|
for _, route := range cycle {
|
|
paths = append(paths, route.canonical)
|
|
if len(route.locations) != 0 {
|
|
related = append(related, route.locations[len(route.locations)-1])
|
|
}
|
|
}
|
|
paths = append(paths, target)
|
|
return Diagnostic{
|
|
Code: CodeImportCycle, Severity: SeverityError,
|
|
Message: "import cycle detected: " + strings.Join(paths, " -> "),
|
|
Primary: &location, Related: related,
|
|
}
|
|
}
|
|
|
|
func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic {
|
|
related := append([]Location(nil), previous.locations...)
|
|
if len(current.locations) > 1 {
|
|
related = append(related, current.locations[:len(current.locations)-1]...)
|
|
}
|
|
return Diagnostic{
|
|
Code: CodeImportRepeated, Severity: SeverityError,
|
|
Message: "configuration source is included by more than one import route",
|
|
Primary: &location, Related: related,
|
|
}
|
|
}
|