ConfigKit/imports.go

package configkit

import (
	"bytes"
	"errors"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
	yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml"
)

const (
	maxImportDepth   = 64
	maxImportSources = 1024
	maxGlobEntries   = 1_000_000
)

type importLimits struct {
	depth   int
	sources int
	bytes   int64
	nodes   int
	entries int
}

func defaultImportLimits() importLimits {
	return importLimits{
		depth: maxImportDepth, sources: maxImportSources,
		bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries,
	}
}

type importDeclaration struct {
	path     string
	glob     string
	optional bool
	location Location
}

type importRoute struct {
	canonical string
	locations []Location
}

type importTraversal struct {
	rootPath string
	root     *os.Root
	limits   importLimits
	rule     *VersionRule

	documents   []*document.Node
	sources     []SourceInfo
	active      map[string]int
	visited     map[string]importRoute
	stack       []importRoute
	usedBytes   int64
	usedNodes   int
	usedEntries int
}

func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
	rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
	if diagnostic != nil {
		return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
	}
	root, err := os.OpenRoot(rootPath)
	if err != nil {
		diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
		diagnostic.Primary = nil
		return nil, sourceInfos(main.source), []Diagnostic{diagnostic}
	}

	mainInfo := cloneSourceInfo(*main.source)
	if mainPath != "" {
		mainInfo.Path = mainPath
	}
	state := importTraversal{
		rootPath: rootPath, root: root, limits: limits, rule: rule,
		documents: make([]*document.Node, 0, 1),
		sources:   []SourceInfo{mainInfo},
		active:    make(map[string]int), visited: make(map[string]importRoute),
		usedBytes: main.bytes, usedNodes: countInputNodes(main.document),
	}
	if diagnostic := state.checkInitialLimits(); diagnostic != nil {
		_ = root.Close()
		return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
	}

	mainIdentity := mainPath
	if mainIdentity == "" {
		mainIdentity = "<selected-source>"
	}
	mainRoute := importRoute{canonical: mainIdentity}
	state.active[mainIdentity] = 0
	state.visited[mainIdentity] = mainRoute
	state.stack = append(state.stack, mainRoute)
	body, declarations, diagnostics := splitImportDirectives(main.document)
	if len(diagnostics) == 0 {
		state.documents = append(state.documents, body)
		diagnostics = state.walkDeclarations(basePath, declarations, 0)
	}

	closeErr := root.Close()
	if len(diagnostics) != 0 {
		return nil, cloneSourceInfos(state.sources), diagnostics
	}
	if closeErr != nil {
		return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
	}
	composed, diagnostics := composeDocuments(state.documents...)
	if composed != nil {
		composed.inputBytes = state.usedBytes
		composed.inputNodes = state.usedNodes
	}
	return composed, cloneSourceInfos(state.sources), diagnostics
}

func validateFilesystemContext(source Source, context *FilesystemContext, main parsedSource) *Diagnostic {
	rootPath, _, _, diagnostic := importFilesystem(source, context, main)
	if diagnostic != nil {
		return diagnostic
	}
	root, err := os.OpenRoot(rootPath)
	if err != nil {
		value := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
		value.Primary = nil
		return &value
	}
	if err := root.Close(); err != nil {
		value := importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)
		return &value
	}
	return nil
}

func filesystemOptionDiagnostics(source Source, context *FilesystemContext) []Diagnostic {
	switch source.kind {
	case sourceNone:
		return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource cannot use filesystem context")}
	case sourceFile:
		if context.BaseDir != "" {
			return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")}
		}
	case sourceBytes, sourceReader:
		if context.BaseDir == "" || context.Root == "" {
			return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem context requires base and root directories")}
		}
		if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) {
			return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "byte and reader filesystem base and root directories must be absolute")}
		}
	}
	return nil
}

func importFilesystem(source Source, context *FilesystemContext, main parsedSource) (rootPath, basePath, mainPath string, diagnostic *Diagnostic) {
	if main.source == nil || main.document == nil {
		value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil)
		return "", "", "", &value
	}

	if source.kind == sourceFile {
		if context != nil && context.BaseDir != "" {
			value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")
			return "", "", "", &value
		}
		canonical, err := canonicalExistingPath(source.path)
		if err != nil {
			value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil)
			return "", "", "", &value
		}
		mainPath = canonical
		basePath = filepath.Dir(canonical)
		rootPath = basePath
		if context != nil && context.Root != "" {
			rootPath, err = canonicalExistingPath(context.Root)
			if err != nil {
				value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
				return "", "", "", &value
			}
		}
	} else {
		if context == nil || context.BaseDir == "" || context.Root == "" {
			value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories")
			return "", "", "", &value
		}
		if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) {
			value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute")
			return "", "", "", &value
		}
		var err error
		basePath, err = canonicalExistingPath(context.BaseDir)
		if err != nil {
			value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
			return "", "", "", &value
		}
		rootPath, err = canonicalExistingPath(context.Root)
		if err != nil {
			value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
			return "", "", "", &value
		}
	}
	if !containedPath(rootPath, basePath) || (mainPath != "" && !containedPath(rootPath, mainPath)) {
		value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
		return "", "", "", &value
	}
	for _, candidate := range []string{rootPath, basePath} {
		info, err := os.Stat(candidate)
		if err != nil || !info.IsDir() {
			value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)
			return "", "", "", &value
		}
	}
	return rootPath, basePath, mainPath, nil
}

func canonicalExistingPath(name string) (string, error) {
	absolute, err := filepath.Abs(name)
	if err != nil {
		return "", err
	}
	return filepath.EvalSymlinks(filepath.Clean(absolute))
}

func containedPath(root, target string) bool {
	relative, err := filepath.Rel(root, target)
	if err != nil || filepath.IsAbs(relative) {
		return false
	}
	return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}

func (s *importTraversal) checkInitialLimits() *Diagnostic {
	if s.limits.sources < 1 {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil)
		return &value
	}
	if s.usedBytes > s.limits.bytes {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil)
		return &value
	}
	if s.usedNodes > s.limits.nodes {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)
		return &value
	}
	return nil
}

func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int) []Diagnostic {
	for _, declaration := range declarations {
		if declaration.glob != "" {
			matches, diagnostics := s.expandImportGlob(basePath, declaration)
			if len(diagnostics) != 0 {
				return diagnostics
			}
			if len(matches) == 0 {
				if declaration.optional {
					continue
				}
				location := declaration.location
				return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)}
			}
			for _, match := range matches {
				matched := declaration
				matched.path = match
				matched.glob = ""
				if diagnostics := s.visitPath(basePath, matched, depth+1); len(diagnostics) != 0 {
					return diagnostics
				}
			}
			continue
		}
		if diagnostics := s.visitPath(basePath, declaration, depth+1); len(diagnostics) != 0 {
			return diagnostics
		}
	}
	return nil
}

func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, depth int) []Diagnostic {
	location := declaration.location
	if depth > s.limits.depth {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
	}
	if !validImportPath(declaration.path) {
		return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)}
	}

	candidate := filepath.Join(basePath, filepath.FromSlash(path.Clean(declaration.path)))
	if !containedPath(s.rootPath, filepath.Clean(candidate)) {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)}
	}
	canonical, err := canonicalExistingPath(candidate)
	if err != nil {
		if declaration.optional && errors.Is(err, fs.ErrNotExist) {
			return nil
		}
		code := CodeImportInvalid
		message := "import target could not be canonicalized"
		if errors.Is(err, fs.ErrNotExist) {
			code, message = CodeImportMissing, "required import target does not exist"
		} else if errors.Is(err, fs.ErrPermission) {
			message = "import target cannot be accessed"
		}
		return []Diagnostic{importDiagnostic(code, message, &location)}
	}
	if !containedPath(s.rootPath, canonical) {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)}
	}

	currentRoute := s.currentRoute(location)
	if activeIndex, ok := s.active[canonical]; ok {
		cycle := append([]importRoute(nil), s.stack[activeIndex:]...)
		return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)}
	}
	if previous, ok := s.visited[canonical]; ok {
		return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)}
	}
	if len(s.sources)+1 > s.limits.sources {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)}
	}

	relative, err := filepath.Rel(s.rootPath, canonical)
	if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)}
	}
	sourceID := "source-" + strconv.Itoa(len(s.sources)+1)
	info := SourceInfo{ID: sourceID, Label: declaration.path, Path: canonical, Layer: LayerBase, ImportedFrom: &location}
	s.sources = append(s.sources, info)
	data, skipped, diagnostic := s.readImport(relative, sourceID, declaration.optional)
	if diagnostic != nil {
		return []Diagnostic{*diagnostic}
	}
	if skipped {
		s.sources = s.sources[:len(s.sources)-1]
		return nil
	}

	root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{
		Depth: maxDocumentDepth, Nodes: maxDocumentNodes,
	})
	if len(issues) != 0 {
		return importParseDiagnostics(sourceID, issues)
	}
	nodeCount := countInputNodes(root)
	if s.usedNodes+nodeCount > s.limits.nodes {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
	}
	s.usedNodes += nodeCount

	body, declarations, diagnostics := splitImportDirectives(root)
	if len(diagnostics) != 0 {
		return diagnostics
	}
	if versionNode := fragmentVersionClaim(body, s.rule); versionNode != nil {
		versionLocation := publicLocation(versionNode.Location)
		return []Diagnostic{{
			Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(),
			Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
		}}
	}

	route := importRoute{canonical: canonical, locations: currentRoute.locations}
	s.documents = append(s.documents, body)
	s.active[canonical] = len(s.stack)
	s.visited[canonical] = route
	s.stack = append(s.stack, route)
	diagnostics = s.walkDeclarations(filepath.Dir(canonical), declarations, depth)
	delete(s.active, canonical)
	s.stack = s.stack[:len(s.stack)-1]
	return diagnostics
}

func (s *importTraversal) readImport(relative, sourceID string, optional bool) ([]byte, bool, *Diagnostic) {
	remaining := s.limits.bytes - s.usedBytes
	limit := min(int64(maxSourceBytes), remaining)
	if limit < 0 {
		limit = 0
	}
	preflight, err := s.root.Stat(relative)
	if err != nil {
		if optional && errors.Is(err, fs.ErrNotExist) {
			return nil, true, nil
		}
		value := classifySourceError(err, sourceID)
		return nil, false, &value
	}
	if !preflight.Mode().IsRegular() {
		value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")
		return nil, false, &value
	}
	file, err := s.root.Open(relative)
	if err != nil {
		if optional && errors.Is(err, fs.ErrNotExist) {
			return nil, true, nil
		}
		value := classifySourceError(err, sourceID)
		return nil, false, &value
	}
	defer func() { _ = file.Close() }()
	info, err := file.Stat()
	if err != nil {
		value := classifySourceError(err, sourceID)
		return nil, false, &value
	}
	if !info.Mode().IsRegular() {
		value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file")
		return nil, false, &value
	}
	if info.Size() > limit {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID})
		return nil, false, &value
	}
	data, err := io.ReadAll(io.LimitReader(file, limit+1))
	if err != nil {
		value := classifySourceError(err, sourceID)
		return nil, false, &value
	}
	if int64(len(data)) > limit {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID})
		return nil, false, &value
	}
	s.usedBytes += int64(len(data))
	return data, false, nil
}

func (s *importTraversal) currentRoute(location Location) importRoute {
	locations := make([]Location, 0, len(s.stack)+1)
	for _, route := range s.stack {
		if len(route.locations) != 0 {
			locations = append(locations, route.locations[len(route.locations)-1])
		}
	}
	locations = append(locations, location)
	return importRoute{locations: locations}
}

func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) {
	body := document.Clone(root)
	var declarations []importDeclaration
	directiveIndex := -1
	for index, entry := range body.Mapping {
		if entry.Key != "_configkit" {
			continue
		}
		directiveIndex = index
		parsed, diagnostics := parseImportDirective(entry.Value, Keys("_configkit"))
		if len(diagnostics) != 0 {
			return nil, nil, diagnostics
		}
		declarations = parsed
		break
	}
	if directiveIndex >= 0 {
		body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...)
	}
	if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 {
		return nil, nil, diagnostics
	}
	return body, declarations, nil
}

func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) {
	if node.Kind != document.Mapping {
		return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)}
	}
	var declarations []importDeclaration
	for _, entry := range node.Mapping {
		fieldPath := directivePath.Key(entry.Key)
		if entry.Key != "imports" {
			return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)}
		}
		if entry.Value.Kind != document.Sequence {
			return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)}
		}
		for index, item := range entry.Value.Sequence {
			declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index))
			if len(diagnostics) != 0 {
				return nil, diagnostics
			}
			declarations = append(declarations, declaration)
		}
	}
	return declarations, nil
}

func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) {
	if node.Kind != document.Mapping {
		return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)}
	}
	declaration := importDeclaration{location: publicLocation(node.Location)}
	seenTarget := false
	for _, entry := range node.Mapping {
		fieldPath := declarationPath.Key(entry.Key)
		switch entry.Key {
		case "path", "glob":
			if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text {
				return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)}
			}
			seenTarget = true
			declaration.location = publicLocation(entry.Value.Location)
			if entry.Key == "path" {
				declaration.path = entry.Value.Text
			} else {
				declaration.glob = entry.Value.Text
			}
		case "optional":
			if entry.Value.Kind != document.Boolean {
				return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)}
			}
			declaration.optional = entry.Value.Text == "true"
		default:
			return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)}
		}
	}
	if !seenTarget {
		return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)}
	}
	return declaration, nil
}

func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic {
	if node.Kind == document.Mapping {
		for _, entry := range node.Mapping {
			childPath := current.Key(entry.Key)
			if entry.Key == "_configkit" {
				location := publicLocation(entry.KeyLocation)
				return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: childPath, Message: "configuration directives are permitted only at the document root", Primary: &location}}
			}
			if diagnostics := nestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 {
				return diagnostics
			}
		}
	} else if node.Kind == document.Sequence {
		for index, child := range node.Sequence {
			if diagnostics := nestedDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 {
				return diagnostics
			}
		}
	}
	return nil
}

func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node {
	if rule == nil {
		return nil
	}
	current := root
	if len(rule.Path.segments) == 0 {
		return current
	}
	for _, segment := range rule.Path.segments {
		if current.Kind != document.Mapping {
			return current
		}
		if segment.kind == pathSegmentIndex {
			return nil
		}
		key := segment.key
		index := documentEntryIndex(current.Mapping, key)
		if index < 0 {
			return nil
		}
		current = current.Mapping[index].Value
	}
	return current
}

func countInputNodes(node *document.Node) int {
	if node == nil {
		return 0
	}
	count := 1
	if node.Kind == document.Mapping {
		count += len(node.Mapping)
		for _, entry := range node.Mapping {
			count += countInputNodes(entry.Value)
		}
	} else if node.Kind == document.Sequence {
		for _, child := range node.Sequence {
			count += countInputNodes(child)
		}
	}
	return count
}

func validImportPath(value string) bool {
	if value == "" || strings.TrimSpace(value) != value || path.IsAbs(value) || strings.HasPrefix(value, "//") {
		return false
	}
	return !hasImportScheme(value)
}

func hasImportScheme(value string) bool {
	colon := strings.IndexByte(value, ':')
	slash := strings.IndexByte(value, '/')
	if colon <= 0 || (slash >= 0 && slash < colon) {
		return false
	}
	for index := 0; index < colon; index++ {
		character := value[index]
		if index == 0 {
			if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') {
				return false
			}
			continue
		}
		if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') &&
			(character < '0' || character > '9') && character != '+' && character != '-' && character != '.' {
			return false
		}
	}
	return true
}

func sourceInfos(source *SourceInfo) []SourceInfo {
	if source == nil {
		return nil
	}
	return []SourceInfo{cloneSourceInfo(*source)}
}

func importParseDiagnostics(sourceID string, issues []yamladapter.Issue) []Diagnostic {
	diagnostics := make([]Diagnostic, len(issues))
	for index, issue := range issues {
		diagnostics[index] = Diagnostic{
			Code: Code(issue.Code), Severity: SeverityError, Message: issue.Message,
			Primary: &Location{SourceID: sourceID, Line: issue.Line, Column: issue.Column},
		}
		for _, related := range issue.Related {
			diagnostics[index].Related = append(diagnostics[index].Related, publicLocation(related))
		}
	}
	return diagnostics
}

func importDiagnostic(code Code, message string, location *Location) Diagnostic {
	return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)}
}

func nodeImportDiagnostic(code Code, path Path, message string, node *document.Node) Diagnostic {
	location := publicLocation(node.Location)
	return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location}
}

func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic {
	paths := make([]string, 0, len(cycle)+1)
	related := make([]Location, 0, len(cycle))
	for _, route := range cycle {
		paths = append(paths, route.canonical)
		if len(route.locations) != 0 {
			related = append(related, route.locations[len(route.locations)-1])
		}
	}
	paths = append(paths, target)
	return Diagnostic{
		Code: CodeImportCycle, Severity: SeverityError,
		Message: "import cycle detected: " + strings.Join(paths, " -> "),
		Primary: &location, Related: related,
	}
}

func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic {
	related := append([]Location(nil), previous.locations...)
	if len(current.locations) > 1 {
		related = append(related, current.locations[:len(current.locations)-1]...)
	}
	return Diagnostic{
		Code: CodeImportRepeated, Severity: SeverityError,
		Message: "configuration source is included by more than one import route",
		Primary: &location, Related: related,
	}
}