diff --git a/README.md b/README.md index 0adfa1b..281ee40 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ An opinionated Go library for applications that use YAML configuration files. ConfigKit is in early implementation. Its current loader accepts one strict YAML -source (or explicit defaults-only input), applies typed defaults, optionally checks -an application schema version, and returns detached metadata and safe diagnostics. +source (or explicit defaults-only input), optionally traverses explicit path imports, +strictly composes their disjoint definitions, applies typed defaults, and can check +an application schema version. It returns detached metadata and safe diagnostics. Applications may perform audited defaulting and normalization through a resolver, then report domain errors and warnings through read-only semantic validation. -Composition, profiles, overrides, secrets, and rendering are still under development. +Glob imports, profiles, overrides, secrets, and rendering are still under development. ```go type Config struct { diff --git a/compose.go b/compose.go index a591cdc..b6feb79 100644 --- a/compose.go +++ b/compose.go @@ -125,6 +125,14 @@ func compositionOrigins(claims []compositionClaim) map[string]Origin { return origins } +func applyCompositionOrigins(decoded *decodedConfiguration, origins map[string]Origin) { + for pointer, origin := range origins { + if _, present := decoded.presence[pointer]; present { + decoded.origins[pointer] = cloneOrigin(origin) + } + } +} + func cloneCompositionClaims(claims []compositionClaim) []compositionClaim { cloned := make([]compositionClaim, len(claims)) for index, claim := range claims { diff --git a/docs/config.md b/docs/config.md index 457826c..8fa61a8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -20,10 +20,11 @@ The zero `Source` is invalid. Constructor argument failures are retained inside opaque source and become safe structured diagnostics when the source is consumed. File sources can observe file changes between consumptions; byte sources cannot. -Each source is limited to 8 MiB. A load will be limited to 64 MiB across all source -and override input when those layers are implemented. Reads request at most the -remaining allowance plus one byte so excess data is rejected rather than silently -truncated. Data returned together with a read error is not accepted as partial input. +Each source is limited to 8 MiB. An imported load is limited to 64 MiB across all +accepted sources, 1,024 accepted sources, 64 active import edges, and 1,000,000 +input nodes in aggregate. Reads request at most the remaining allowance plus one byte +so excess data is rejected rather than silently truncated. Data returned together +with a read error is not accepted as partial input. ## Single-source loading @@ -36,21 +37,22 @@ schema-version checking. Its result is detached before input is applied. The current lifecycle is: ```text -validate options → parse one source → reject reserved directives → build schema - → check the optional application version → create and clone defaults +validate options → parse the selected source → traverse/compose enabled imports + → build schema → check the optional application version → create and clone defaults → strictly decode → recorded application resolution → semantic validation → snapshot configuration and metadata ``` -`File`, `Bytes`, and `Reader` each contribute one base source with the deterministic -ID `source-1`. `NoSource` skips parsing, supplies an absent version to a configured -version checker, contributes no source, and continues through defaults, decode, and -projection. A missing file remains an error; it is never interpreted as no input. +`File`, `Bytes`, and `Reader` each contribute the first base source with deterministic +ID `source-1`; enabled imports continue in depth-first order as `source-2` onward. +`NoSource` skips parsing, supplies an absent version to a configured version checker, +contributes no source, and continues through defaults, decode, and projection. A +missing file remains an error; it is never interpreted as no input. The current `Options` boundary accepts a defaults factory, `VersionRule`, resolver, -and semantic validator. Imports, active profiles, filesystem context, overrides, -and dynamic secret classification return an explicit -`options.unavailable` error because their lifecycle stages are not implemented. +semantic validator, explicit path imports, and import filesystem context. Active +profiles, standalone filesystem context, overrides, and dynamic secret classification +return `options.unavailable` because their lifecycle stages are not implemented. The reserved profile spelling `default` selects no active profile. `NoSource` additionally rejects imports, active profiles, and filesystem context as invalid combinations. Input containing a reserved `_configkit` mapping key is rejected while @@ -152,10 +154,8 @@ live services as a substitute for configuration validation. ## Strict document composition -ConfigKit's private document layer can strictly compose parsed base documents. The -operation is available for file-graph integration but is not yet reachable through -the public single-source `Load` lifecycle. Application defaults do not participate -in this operation. +ConfigKit strictly composes parsed base documents when `Options.Imports` is true. +Application defaults do not participate in this operation. | Existing node | Incoming node | Result | | --- | --- | --- | @@ -180,6 +180,44 @@ traversal order. Traversal order never authorizes replacement. Composition clone the resulting tree and its ordered claim records rather than mutating or aliasing an input document. +## Explicit path imports + +Imports are opt-in. With `Options.Imports` enabled, a document may reserve its root +`_configkit` mapping for an `imports` sequence. ConfigKit removes that mapping before +application decoding. Each sequence item is a mapping with exactly one nonempty +string `path` or `glob` plus an optional boolean `optional`; all other shapes and +fields are `import.invalid`. `glob` is recognized but temporarily returns +`import.unsupported` until bounded glob expansion is implemented. A `_configkit` +key below the root is invalid. With imports disabled, any occurrence remains +`import.disabled`. + +Path declarations use slash-separated relative paths. Absolute paths and network +URLs are invalid. Each path resolves from its declaring file's canonical directory, +and files are traversed depth-first in declaration order: a file contributes its own +body before its imports. Order stabilizes source records and diagnostics but never +permits replacement. Required missing paths fail; `optional: true` suppresses only +target nonexistence, not invalid paths, containment failures, directories, +permissions, parse errors, or composition errors. + +The configuration root defaults to a selected file's canonical directory. A file +source may supply `FilesystemContext.Root` to widen that root but may not supply +`BaseDir`. Byte and reader sources must supply both `BaseDir` and `Root` as absolute +existing directories when imports are enabled; their diagnostic labels, including +URI-looking labels, never imply filesystem paths. The base and selected file must be +inside the canonical root. ConfigKit canonicalizes symlinks, checks containment with +relative paths, and opens the canonical root-relative target through a library-owned +root handle. Contained relative or absolute symlink targets are accepted; escapes +are rejected. Canonical paths define identity, so hard links at distinct paths remain +distinct sources. + +An active-stack repeat is `import.cycle` and reports the complete cycle's import +sites. A previously visited canonical path is `import.repeated` and reports both +inclusion routes, including diamonds and aliases. Each accepted file is read once. +Imported fragments may contain their own root directives, but may not claim the +application's schema-version path or replace one of its ancestors; the version must +come from the selected source. No directory is scanned implicitly, including one +named `conf.d`, and ConfigKit performs no remote fetching. + ## YAML documents The private YAML boundary currently accepts exactly one nonempty document whose root diff --git a/docs/development.md b/docs/development.md index 9851b3b..18981c7 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,8 +2,9 @@ This is the contributor entry point for ConfigKit. Read the task-specific owners below before making changes. The repository is in early implementation: its public -loader currently covers one strict source, typed defaults, schema-version checks, -recorded application resolution, semantic validation, and metadata. The +loader currently covers strict selected sources, explicit path-import graphs, +composition, typed defaults, schema-version checks, recorded application resolution, +semantic validation, and metadata. The [component overview](internal/overview.md) owns the implementation inventory. ## Initial orientation diff --git a/docs/internal/extraction.md b/docs/internal/extraction.md index e0c1c20..fed25fb 100644 --- a/docs/internal/extraction.md +++ b/docs/internal/extraction.md @@ -112,6 +112,18 @@ Assigned to explicit import traversal and glob handling: traversal, canonical containment, cycle/diamond routes, and fixed source IDs. Canonical paths—not `os.SameFile`—define identity, so hard links remain distinct. +Path-import implementation record (2026-09-06): `splitImportDirectives`, +`parseImportDeclaration`, `importFilesystem`, `importTraversal.visitPath`, and +`loadImportedDocuments` in `imports.go` adapt the assigned envelope, loading, source, +and failure mechanics. The corresponding ConfigKit tests adapt the assigned +declaration-order, disjoint composition, keyed-entry, all-source conflict, unsafe +target, alias, imported parse/schema, and relative-resolution cases. ConfigKit adds +recursive bounded DFS, `os.OpenRoot` confinement, canonical symlink identity, +cycle/diamond route locations, optional-path semantics, aggregate budgets, and +fragment version-placement enforcement. The planned glob syntax is parsed but +remains explicitly unavailable for its next owning work; hard links intentionally +replace Narratio's same-file rejection with path identity. + ### Profiles and overlay Assigned to selected-profile overlay: diff --git a/docs/internal/overview.md b/docs/internal/overview.md index bbbd679..8b3b411 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -20,18 +20,18 @@ module and package documentation boundary. Its implemented public primitives are - strict schema-driven decoding over detached defaults in `decode.go`, including mapping/sequence semantics, null and numeric policy, safe codec errors, presence, baseline/final trees, and initial default/input origins; and -- public single-source lifecycle coordination in `load.go`, version-report contexts - in `validation.go`, and detached base/default metadata assembly in - `load_metadata.go`; and +- public lifecycle coordination in `load.go`, bounded canonical path-import DFS in + `imports.go`, version-report contexts in `validation.go`, and detached + base/default metadata assembly in `load_metadata.go`; and - recorded application defaulting and normalization in `resolution.go`, including explicitness queries, helper verification, structural mutation audit, coalesced intent, private before/after snapshots, and resolution metadata integration; and - post-resolution schema-aware report collection and validator mutation detection in - `validation.go`, completing the public single-source callback lifecycle. + `validation.go`, completing the public callback lifecycle after composed input. The [configuration contract](../config.md) owns implemented input, schema, decoding, -single-source lifecycle, version checking, recorded resolution, semantic validation, -metadata, and ownership behavior. +loading lifecycle, path imports, composition, version checking, recorded resolution, +semantic validation, metadata, and ownership behavior. The source-extraction inventory records the application foundations assigned to future implementation work: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index bed9687..b879a69 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -850,7 +850,38 @@ Acceptance: ## Stage 11 — Implement explicit file-import traversal -Status: pending. +Status: complete. + +Completion evidence (2026-09-06): + +- Added strict recursive `_configkit.imports` parsing with exactly one `path` or + recognized `glob`, optional booleans, unknown-shape rejection, root-only placement, + and detached directive removal. Glob declarations return the deliberate temporary + `import.unsupported` gate for the next owning work. +- Implemented declaration-order depth-first path traversal with canonical symlink + identity, relative containment checks, and canonical root-relative access through a + ConfigKit-owned `os.Root`. File inputs infer their canonical base/root; labeled + byte and reader inputs require explicit absolute base/root context and never infer + paths from URI-like labels. +- Kept active and visited indexes separate, reporting complete cycle import sites and + both diamond/alias inclusion routes. Canonical aliases repeat, while hard links at + distinct paths remain separate sources. Required missing paths, directories, + unreadable targets, escapes, and canonicalization failures remain errors; optional + suppresses only nonexistence. +- Assigned deterministic source IDs while reading each accepted source once, enforced + per-source and aggregate bytes, aggregate input nodes, source count, and active-edge + depth, and retained import-site/source records on failures. Imported fragments are + rejected when they claim the application version path or a non-mapping ancestor, + before strict composition. +- Wired composed input and multi-location origins through `Load`, version checking, + strict decoding, resolution, validation, and base metadata. Temporary-directory + tests cover nested/list-order traversal, composition conflicts, version placement, + directives, optional/required paths, explicit roots, cycles, diamonds, symlinks, + hard links, parse/decode attribution, file types, context rules, and reduced limit + boundaries. The [current path-import contract](../config.md#explicit-path-imports) + owns implemented behavior; glob expansion remains pending. Verified `go test ./...`, + `go test -race ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, and + `git diff --check` with Go 1.26.7 on linux/amd64. Read: feature [import graph](initial.md#explicit-imports-and-source-identity). diff --git a/docs/roadmap/initial.md b/docs/roadmap/initial.md index bc47786..ca80fe5 100644 --- a/docs/roadmap/initial.md +++ b/docs/roadmap/initial.md @@ -426,14 +426,14 @@ configuration is never returned as a successful, usable result. Source ownership, the supported YAML subset, scalar grammar, fixed input limits, and safe failure behavior are implemented. The [current configuration input contract](../config.md) now owns those definitions, including typed scalar -conversion through the current single-source lifecycle. +conversion through the current loading lifecycle. ## Paths and presence Semantic path construction, JSON Pointer parsing, and base/default metadata population are implemented. Their current contracts are owned by the Go documentation in [`path.go`](../../path.go), [`metadata.go`](../../metadata.go), -and the [single-source configuration contract](../config.md#base-default-metadata). +and the [current configuration contract](../config.md#base-default-metadata). Layered replacement and resolution behavior below remains target lifecycle work. Distinguish absent, explicitly null, explicitly zero/false/empty, and other @@ -475,62 +475,30 @@ the surviving sequence, not an attempted correspondence with previous indices. ## Explicit imports and source identity -Imports are opt-in through the loading API. Reserve the root `_configkit` mapping -for library directives and remove it before application decoding. Initially its -only permitted field is `imports`, a sequence of declarations. Illustrative -fragment: +Explicit path-import traversal, source identity, containment, budgets, recursive +directives, and the temporary glob gate are implemented and owned by the +[current configuration contract](../config.md#explicit-path-imports). Bounded glob +expansion remains target behavior. Illustrative future fragment: ```yaml _configkit: imports: - - path: fragments/server.yml - glob: fragments/outputs/*.yml optional: true ``` -Each declaration has exactly one nonempty `path` or `glob`; `optional` is a bool -and defaults to false. Unknown directive fields are errors. When imports are -disabled, the presence of `_configkit` is an error, not an ignored directive. - -- Relative paths resolve from the declaring file's directory. Absolute import - paths and network URLs are rejected. - Expand declarations in list order, sorting each glob's matches lexicographically by slash-separated relative path, with no locale sorting. Use the slash-based [Go `path.Match` grammar](https://pkg.go.dev/path#Match) (`*`, `?`, character classes, and backslash escapes), with recursive `**` syntax forbidden. Backslashes escape pattern characters, not directory separators. -- Traverse depth-first: process a file's own contribution, then its declarations - in order. Ordering stabilizes source lists and diagnostics; it never permits - duplicate definitions to override one another. -- A required missing path or unmatched glob is an error. Optional suppresses - only a nonexistent target or zero matches. Invalid patterns, permission +- A required unmatched glob is an error. Optional suppresses only zero matches. + Invalid patterns, permission failures, directories, and invalid matched documents remain errors. -- The configuration root defaults to the main file's directory; callers may - explicitly supply another root. Resolve symlinks and cleaned absolute paths - for identity and containment. Imports and profiles must stay inside that - canonical root. Retain declared paths and import-site locations for messages. - Relative and absolute symlink targets are permitted when their canonical target - is contained; the spelling of a symlink target alone does not determine containment. -- An active-stack repeat is a cycle; report the complete cycle and import sites. - A previously visited canonical file outside the active stack is a repeated - import error, including diamond graphs and overlapping glob matches. Report - both inclusion routes. Hard links at distinct canonical paths remain distinct - sources; this is path identity, not inode deduplication. -- Read each accepted source once per load. Do not promise an atomic snapshot of - a filesystem being modified concurrently. Bound input bytes, node nesting, - import depth, and source count with documented limits and structured errors. - Excess input is an error, never an implicitly truncated document. Glob traversal - counts examined directory entries, including nonmatches, against its work budget; - only accepted matches contribute to the sorted source order. - -Bytes/reader sources require a diagnostic label. Imports and profiles for these -sources require an explicit filesystem base directory and root; never infer a -filesystem location from a label such as an S3 URI. Without that context they -remain unavailable. ConfigKit does not fetch remote imports. - -Debian-style automatic `conf.d` discovery is deferred. A caller may explicitly -import a file or glob inside a directory named `conf.d`, but that name has no -special behavior. Do not scan adjacent directories implicitly. +- Glob traversal counts examined directory entries, including nonmatches, against + its separate work budget; only accepted matches contribute to sorted source order. + Expansion remains pattern-directed and must preserve the current path-import + identity, containment, repetition, and depth-first traversal rules. ## Profiles @@ -650,11 +618,12 @@ Automatic migration is deferred. ## Consumer API design -The core declarations and single-source loading subset are implemented; their -current contract is owned by [configuration input](../config.md#single-source-loading) -and Go reference comments. Remaining declarations and behavior in this section are -target design, not implemented Go reference documentation or a frozen naming -commitment. Declarations omit method bodies and private fields. +The core declarations, callback lifecycle, strict composition, and explicit path +imports are implemented; their current contract is owned by +[configuration input](../config.md) and Go reference comments. Remaining declarations +and behavior in this section are target design, not implemented Go reference +documentation or a frozen naming commitment. Declarations omit method bodies and +private fields. Consumer snippets assume the indicated standard-library imports and a `configkit` import using the module path selected in the [implementation plan](implementation.md#implementation-decisions). They are illustrative target examples, not currently runnable examples. The @@ -664,11 +633,10 @@ and verification against consumer models. ### Entry point and reusable application definition `Load`, `Definition`, `Options`, `FilesystemContext`, `Override`, `VersionRule`, -`Version`, `Validation`, and `Result` are declared. The current loader accepts only -the defaults and version-checking subset described by the -[single-source contract](../config.md#single-source-loading); later-feature fields -fail explicitly rather than being ignored. The complete lifecycle and option -behavior below remains the target as its owning stages land. +`Version`, `Validation`, and `Result` are declared. The current loader supports the +callback and path-import subset described by the [current contract](../config.md); +later profile, override, secret, and rendering behavior below remains target work and +unavailable options fail explicitly rather than being ignored. ```go func Load[T any](source Source, options ...Options[T]) (*Result[T], error) @@ -710,16 +678,11 @@ result. Callbacks must not retain these contexts or configuration pointers; shar closures must be safe for concurrent calls if the application loads concurrently. ConfigKit does not synchronize application-owned captured state. -`Source` and its constructors are implemented; [`source.go`](../../source.go) and -the [configuration input contract](../config.md#sources) own their current behavior. -The loading and filesystem-option behavior below remains target work. -Applications must use non-secret source labels because labels appear in diagnostics -and source reports. -Filesystem context follows the [source rules](#explicit-imports-and-source-identity): -a file's own directory supplies its base, and an explicit context can set its -root. Supplying `BaseDir` for a file is an options error; bytes/readers require -both absolute `BaseDir` and `Root` when imports or profiles are used. A base must -be inside its root. These options do not change application field path meanings. +`Source`, its constructors, and import filesystem context are implemented; +[`source.go`](../../source.go) and the [configuration input contract](../config.md#sources) +own their current behavior. Applications must use non-secret source labels because +labels appear in diagnostics and source reports. Profile-related filesystem behavior +remains target work. Filesystem options do not change application field path meanings. `NoSource()` explicitly selects defaults-only loading. It contributes no document or file source entry and marks no fields explicit by itself; CLI overrides still @@ -1030,7 +993,7 @@ applications own secret-producing transformations not described by those records ### Results, errors, and inspection `Result.Config`, `Metadata`, `Sources`, and `Diagnostics`, together with -`LoadError` and `WriteDiagnostics`, are implemented at the single-source boundary. +`LoadError` and `WriteDiagnostics`, are implemented at the current loading boundary. The [current loading contract](../config.md#single-source-loading) and Go reference comments own their behavior. Effective rendering and richer layered inspection below remain target work. diff --git a/imports.go b/imports.go new file mode 100644 index 0000000..fce6668 --- /dev/null +++ b/imports.go @@ -0,0 +1,582 @@ +package configkit + +import ( + "bytes" + "errors" + "io" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" + yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml" +) + +const ( + maxImportDepth = 64 + maxImportSources = 1024 +) + +type importLimits struct { + depth int + sources int + bytes int64 + nodes int +} + +func defaultImportLimits() importLimits { + return importLimits{ + depth: maxImportDepth, sources: maxImportSources, + bytes: maxTotalBytes, nodes: maxDocumentNodes, + } +} + +type importDeclaration struct { + path string + glob string + optional bool + location Location +} + +type importRoute struct { + canonical string + locations []Location +} + +type importTraversal struct { + rootPath string + root *os.Root + limits importLimits + rule *VersionRule + + documents []*document.Node + sources []SourceInfo + active map[string]int + visited map[string]importRoute + stack []importRoute + usedBytes int64 + usedNodes int +} + +func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) + if diagnostic != nil { + return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} + } + root, err := os.OpenRoot(rootPath) + if err != nil { + return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root could not be opened", nil)} + } + + mainInfo := cloneSourceInfo(*main.source) + if mainPath != "" { + mainInfo.Path = mainPath + } + state := importTraversal{ + rootPath: rootPath, root: root, limits: limits, rule: rule, + documents: make([]*document.Node, 0, 1), + sources: []SourceInfo{mainInfo}, + active: make(map[string]int), visited: make(map[string]importRoute), + usedBytes: main.bytes, usedNodes: countInputNodes(main.document), + } + if diagnostic := state.checkInitialLimits(); diagnostic != nil { + _ = root.Close() + return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} + } + + mainIdentity := mainPath + if mainIdentity == "" { + mainIdentity = "" + } + mainRoute := importRoute{canonical: mainIdentity} + state.active[mainIdentity] = 0 + state.visited[mainIdentity] = mainRoute + state.stack = append(state.stack, mainRoute) + body, declarations, diagnostics := splitImportDirectives(main.document) + if len(diagnostics) == 0 { + state.documents = append(state.documents, body) + diagnostics = state.walkDeclarations(basePath, declarations, 0) + } + + closeErr := root.Close() + if len(diagnostics) != 0 { + return nil, cloneSourceInfos(state.sources), diagnostics + } + if closeErr != nil { + return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root could not be closed", nil)} + } + composed, diagnostics := composeDocuments(state.documents...) + return composed, cloneSourceInfos(state.sources), diagnostics +} + +func importFilesystem(source Source, context *FilesystemContext, main parsedSource) (rootPath, basePath, mainPath string, diagnostic *Diagnostic) { + if main.source == nil || main.document == nil { + value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil) + return "", "", "", &value + } + + if source.kind == sourceFile { + if context != nil && context.BaseDir != "" { + value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory") + return "", "", "", &value + } + canonical, err := canonicalExistingPath(source.path) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil) + return "", "", "", &value + } + mainPath = canonical + basePath = filepath.Dir(canonical) + rootPath = basePath + if context != nil && context.Root != "" { + rootPath, err = canonicalExistingPath(context.Root) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) + return "", "", "", &value + } + } + } else { + if context == nil || context.BaseDir == "" || context.Root == "" { + value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories") + return "", "", "", &value + } + if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) { + value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute") + return "", "", "", &value + } + var err error + basePath, err = canonicalExistingPath(context.BaseDir) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil) + return "", "", "", &value + } + rootPath, err = canonicalExistingPath(context.Root) + if err != nil { + value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil) + return "", "", "", &value + } + } + if !containedPath(rootPath, basePath) || (mainPath != "" && !containedPath(rootPath, mainPath)) { + value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil) + return "", "", "", &value + } + for _, candidate := range []string{rootPath, basePath} { + info, err := os.Stat(candidate) + if err != nil || !info.IsDir() { + value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil) + return "", "", "", &value + } + } + return rootPath, basePath, mainPath, nil +} + +func canonicalExistingPath(name string) (string, error) { + absolute, err := filepath.Abs(name) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(filepath.Clean(absolute)) +} + +func containedPath(root, target string) bool { + relative, err := filepath.Rel(root, target) + if err != nil || filepath.IsAbs(relative) { + return false + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func (s *importTraversal) checkInitialLimits() *Diagnostic { + if s.limits.sources < 1 { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil) + return &value + } + if s.usedBytes > s.limits.bytes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil) + return &value + } + if s.usedNodes > s.limits.nodes { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil) + return &value + } + return nil +} + +func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int) []Diagnostic { + for _, declaration := range declarations { + if declaration.glob != "" { + location := declaration.location + return []Diagnostic{importDiagnostic(CodeImportUnsupported, "glob imports are not available in this release boundary", &location)} + } + if diagnostics := s.visitPath(basePath, declaration, depth+1); len(diagnostics) != 0 { + return diagnostics + } + } + return nil +} + +func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, depth int) []Diagnostic { + location := declaration.location + if depth > s.limits.depth { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} + } + if !validImportPath(declaration.path) { + return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)} + } + + candidate := filepath.Join(basePath, filepath.FromSlash(path.Clean(declaration.path))) + if !containedPath(s.rootPath, filepath.Clean(candidate)) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} + } + canonical, err := canonicalExistingPath(candidate) + if err != nil { + if declaration.optional && errors.Is(err, fs.ErrNotExist) { + return nil + } + code := CodeImportInvalid + message := "import target could not be canonicalized" + if errors.Is(err, fs.ErrNotExist) { + code, message = CodeImportMissing, "required import target does not exist" + } else if errors.Is(err, fs.ErrPermission) { + message = "import target cannot be accessed" + } + return []Diagnostic{importDiagnostic(code, message, &location)} + } + if !containedPath(s.rootPath, canonical) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} + } + + currentRoute := s.currentRoute(location) + if activeIndex, ok := s.active[canonical]; ok { + cycle := append([]importRoute(nil), s.stack[activeIndex:]...) + return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} + } + if previous, ok := s.visited[canonical]; ok { + return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} + } + if len(s.sources)+1 > s.limits.sources { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)} + } + + relative, err := filepath.Rel(s.rootPath, canonical) + if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)} + } + sourceID := "source-" + strconv.Itoa(len(s.sources)+1) + info := SourceInfo{ID: sourceID, Label: declaration.path, Path: canonical, Layer: LayerBase, ImportedFrom: &location} + s.sources = append(s.sources, info) + data, skipped, diagnostic := s.readImport(relative, sourceID, declaration.optional) + if diagnostic != nil { + return []Diagnostic{*diagnostic} + } + if skipped { + s.sources = s.sources[:len(s.sources)-1] + return nil + } + + root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{ + Depth: maxDocumentDepth, Nodes: maxDocumentNodes, + }) + if len(issues) != 0 { + return importParseDiagnostics(sourceID, issues) + } + nodeCount := countInputNodes(root) + if s.usedNodes+nodeCount > s.limits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} + } + s.usedNodes += nodeCount + + body, declarations, diagnostics := splitImportDirectives(root) + if len(diagnostics) != 0 { + return diagnostics + } + if versionNode := fragmentVersionClaim(body, s.rule); versionNode != nil { + versionLocation := publicLocation(versionNode.Location) + return []Diagnostic{{ + Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(), + Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation, + }} + } + + route := importRoute{canonical: canonical, locations: currentRoute.locations} + s.documents = append(s.documents, body) + s.active[canonical] = len(s.stack) + s.visited[canonical] = route + s.stack = append(s.stack, route) + diagnostics = s.walkDeclarations(filepath.Dir(canonical), declarations, depth) + delete(s.active, canonical) + s.stack = s.stack[:len(s.stack)-1] + return diagnostics +} + +func (s *importTraversal) readImport(relative, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { + remaining := s.limits.bytes - s.usedBytes + limit := min(int64(maxSourceBytes), remaining) + if limit < 0 { + limit = 0 + } + file, err := s.root.Open(relative) + if err != nil { + if optional && errors.Is(err, fs.ErrNotExist) { + return nil, true, nil + } + value := classifySourceError(err, sourceID) + return nil, false, &value + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + value := classifySourceError(err, sourceID) + return nil, false, &value + } + if !info.Mode().IsRegular() { + value := sourceDiagnostic(CodeSourceNotRegular, sourceID, "source is not a regular file") + return nil, false, &value + } + if info.Size() > limit { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID}) + return nil, false, &value + } + data, err := io.ReadAll(io.LimitReader(file, limit+1)) + if err != nil { + value := classifySourceError(err, sourceID) + return nil, false, &value + } + if int64(len(data)) > limit { + value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit", &Location{SourceID: sourceID}) + return nil, false, &value + } + s.usedBytes += int64(len(data)) + return data, false, nil +} + +func (s *importTraversal) currentRoute(location Location) importRoute { + locations := make([]Location, 0, len(s.stack)+1) + for _, route := range s.stack { + if len(route.locations) != 0 { + locations = append(locations, route.locations[len(route.locations)-1]) + } + } + locations = append(locations, location) + return importRoute{locations: locations} +} + +func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { + body := document.Clone(root) + var declarations []importDeclaration + directiveIndex := -1 + for index, entry := range body.Mapping { + if entry.Key != "_configkit" { + continue + } + directiveIndex = index + parsed, diagnostics := parseImportDirective(entry.Value, Keys("_configkit")) + if len(diagnostics) != 0 { + return nil, nil, diagnostics + } + declarations = parsed + break + } + if directiveIndex >= 0 { + body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...) + } + if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 { + return nil, nil, diagnostics + } + return body, declarations, nil +} + +func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) { + if node.Kind != document.Mapping { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)} + } + var declarations []importDeclaration + for _, entry := range node.Mapping { + fieldPath := directivePath.Key(entry.Key) + if entry.Key != "imports" { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)} + } + if entry.Value.Kind != document.Sequence { + return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)} + } + for index, item := range entry.Value.Sequence { + declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index)) + if len(diagnostics) != 0 { + return nil, diagnostics + } + declarations = append(declarations, declaration) + } + } + return declarations, nil +} + +func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) { + if node.Kind != document.Mapping { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} + } + declaration := importDeclaration{location: publicLocation(node.Location)} + seenTarget := false + for _, entry := range node.Mapping { + fieldPath := declarationPath.Key(entry.Key) + switch entry.Key { + case "path", "glob": + if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)} + } + seenTarget = true + declaration.location = publicLocation(entry.Value.Location) + if entry.Key == "path" { + declaration.path = entry.Value.Text + } else { + declaration.glob = entry.Value.Text + } + case "optional": + if entry.Value.Kind != document.Boolean { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)} + } + declaration.optional = entry.Value.Text == "true" + default: + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} + } + } + if !seenTarget { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)} + } + return declaration, nil +} + +func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { + if node.Kind == document.Mapping { + for _, entry := range node.Mapping { + childPath := current.Key(entry.Key) + if entry.Key == "_configkit" { + location := publicLocation(entry.KeyLocation) + return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: childPath, Message: "configuration directives are permitted only at the document root", Primary: &location}} + } + if diagnostics := nestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { + return diagnostics + } + } + } else if node.Kind == document.Sequence { + for index, child := range node.Sequence { + if diagnostics := nestedDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { + return diagnostics + } + } + } + return nil +} + +func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node { + if rule == nil { + return nil + } + current := root + if len(rule.Path.segments) == 0 { + return current + } + for _, segment := range rule.Path.segments { + if current.Kind != document.Mapping { + return current + } + if segment.kind == pathSegmentIndex { + return nil + } + key := segment.key + index := documentEntryIndex(current.Mapping, key) + if index < 0 { + return nil + } + current = current.Mapping[index].Value + } + return current +} + +func countInputNodes(node *document.Node) int { + if node == nil { + return 0 + } + count := 1 + if node.Kind == document.Mapping { + count += len(node.Mapping) + for _, entry := range node.Mapping { + count += countInputNodes(entry.Value) + } + } else if node.Kind == document.Sequence { + for _, child := range node.Sequence { + count += countInputNodes(child) + } + } + return count +} + +func validImportPath(value string) bool { + if value == "" || strings.TrimSpace(value) != value || path.IsAbs(value) || strings.HasPrefix(value, "//") { + return false + } + parsed, err := url.Parse(value) + return err == nil && parsed.Scheme == "" +} + +func sourceInfos(source *SourceInfo) []SourceInfo { + if source == nil { + return nil + } + return []SourceInfo{cloneSourceInfo(*source)} +} + +func importParseDiagnostics(sourceID string, issues []yamladapter.Issue) []Diagnostic { + diagnostics := make([]Diagnostic, len(issues)) + for index, issue := range issues { + diagnostics[index] = Diagnostic{ + Code: Code(issue.Code), Severity: SeverityError, Message: issue.Message, + Primary: &Location{SourceID: sourceID, Line: issue.Line, Column: issue.Column}, + } + for _, related := range issue.Related { + diagnostics[index].Related = append(diagnostics[index].Related, publicLocation(related)) + } + } + return diagnostics +} + +func importDiagnostic(code Code, message string, location *Location) Diagnostic { + return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)} +} + +func nodeImportDiagnostic(code Code, path Path, message string, node *document.Node) Diagnostic { + location := publicLocation(node.Location) + return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location} +} + +func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic { + paths := make([]string, 0, len(cycle)+1) + related := make([]Location, 0, len(cycle)) + for _, route := range cycle { + paths = append(paths, route.canonical) + if len(route.locations) != 0 { + related = append(related, route.locations[len(route.locations)-1]) + } + } + paths = append(paths, target) + return Diagnostic{ + Code: CodeImportCycle, Severity: SeverityError, + Message: "import cycle detected: " + strings.Join(paths, " -> "), + Primary: &location, Related: related, + } +} + +func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic { + related := append([]Location(nil), previous.locations...) + if len(current.locations) > 1 { + related = append(related, current.locations[:len(current.locations)-1]...) + } + return Diagnostic{ + Code: CodeImportRepeated, Severity: SeverityError, + Message: "configuration source is included by more than one import route", + Primary: &location, Related: related, + } +} diff --git a/imports_test.go b/imports_test.go new file mode 100644 index 0000000..7b60db7 --- /dev/null +++ b/imports_test.go @@ -0,0 +1,455 @@ +package configkit + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" +) + +type importTestConfig struct { + Settings struct { + Root string `yaml:"root"` + First string `yaml:"first"` + Nested string `yaml:"nested"` + Second string `yaml:"second"` + } `yaml:"settings"` + Version string `yaml:"version"` +} + +func TestLoadTraversesPathImportsDepthFirstAndComposesOrigins(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `_configkit: + imports: + - path: first.yml + - path: second.yml +settings: + root: root +`) + firstPath := writeImportFile(t, directory, "first.yml", `_configkit: + imports: + - path: nested/third.yml +settings: + first: first +`) + thirdPath := writeImportFile(t, directory, "nested/third.yml", "settings:\n nested: nested\n") + secondPath := writeImportFile(t, directory, "second.yml", "settings:\n second: second\n") + + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil { + t.Fatal(err) + } + if got := result.Config.Settings; got.Root != "root" || got.First != "first" || got.Nested != "nested" || got.Second != "second" { + t.Fatalf("composed settings = %#v", got) + } + sources := result.Sources() + wantPaths := []string{canonicalTestPath(t, mainPath), canonicalTestPath(t, firstPath), canonicalTestPath(t, thirdPath), canonicalTestPath(t, secondPath)} + gotPaths := make([]string, len(sources)) + for index, source := range sources { + gotPaths[index] = source.Path + if source.ID != "source-"+strconv.Itoa(index+1) { + t.Errorf("source %d ID = %q", index, source.ID) + } + if index > 0 && source.ImportedFrom == nil { + t.Errorf("source %d has no import location", index) + } + } + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Fatalf("source paths = %#v, want depth-first %#v", gotPaths, wantPaths) + } + info, ok := result.Metadata().Lookup(Keys("settings")) + if !ok || !info.Explicit || len(info.Origin.Locations) != 4 { + t.Fatalf("settings metadata = %#v, %v", info, ok) + } +} + +func TestLoadRejectsImportConflictsAndFragmentVersions(t *testing.T) { + t.Run("all duplicate claimants", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `_configkit: + imports: + - path: one.yml + - path: two.yml + - path: three.yml +`) + for index, name := range []string{"one.yml", "two.yml", "three.yml"} { + writeImportFile(t, directory, name, "settings:\n root: "+string(rune('a'+index))+"\n") + } + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/settings/root" || diagnostic.Primary == nil || len(diagnostic.Related) != 2 { + t.Fatalf("conflict diagnostic = %#v", diagnostic) + } + }) + + versionRule := &VersionRule{Path: Keys("settings", "version"), Check: func(Version, *Validation) error { return nil }} + for _, test := range []struct { + name string + fragment string + }{ + {name: "exact", fragment: "settings:\n version: v2\n"}, + {name: "ancestor scalar", fragment: "settings: blocked\n"}, + } { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports:\n - path: fragment.yml\n") + writeImportFile(t, directory, "fragment.yml", test.fragment) + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true, Definition: Definition[importTestConfig]{Version: versionRule}}, CodeImportVersion) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Path.String() != "/settings/version" || diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" { + t.Fatalf("version placement diagnostic = %#v", diagnostic) + } + }) + } +} + +func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) { + tests := []struct { + name string + body string + code Code + }{ + {name: "directive kind", body: "_configkit: []\n", code: CodeImportInvalid}, + {name: "unknown directive field", body: "_configkit: {unknown: true}\n", code: CodeImportInvalid}, + {name: "imports kind", body: "_configkit: {imports: {}}\n", code: CodeImportInvalid}, + {name: "declaration kind", body: "_configkit: {imports: [file.yml]}\n", code: CodeImportInvalid}, + {name: "missing target", body: "_configkit: {imports: [{optional: true}]}\n", code: CodeImportInvalid}, + {name: "two targets", body: "_configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, + {name: "unknown declaration field", body: "_configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, + {name: "optional kind", body: "_configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, + {name: "glob gate", body: "_configkit: {imports: [{glob: '*.yml'}]}\n", code: CodeImportUnsupported}, + {name: "nested directive", body: "nested: {_configkit: {}}\n", code: CodeImportInvalid}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", test.body) + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) + }) + } +} + +func TestLoadImportMissingContainmentAndRegularFileRules(t *testing.T) { + t.Run("optional missing", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `_configkit: + imports: + - path: missing.yml + optional: true +settings: {root: retained} +`) + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || result.Config.Settings.Root != "retained" || len(result.Sources()) != 1 { + t.Fatalf("optional result = %#v, %v", result, err) + } + }) + + tests := []struct { + name string + path string + setup func(*testing.T, string) + code Code + }{ + {name: "required missing", path: "missing.yml", code: CodeImportMissing}, + {name: "parent traversal", path: "../outside.yml", setup: func(t *testing.T, directory string) { + writeImportFile(t, filepath.Dir(directory), "outside.yml", "settings: {root: outside}\n") + }, code: CodeImportOutsideRoot}, + {name: "directory", path: "directory", setup: func(t *testing.T, directory string) { + if err := os.Mkdir(filepath.Join(directory, "directory"), 0o755); err != nil { + t.Fatal(err) + } + }, code: CodeSourceNotRegular}, + {name: "absolute", path: "/tmp/configkit.yml", code: CodeImportInvalid}, + {name: "network URL", path: "https://example.invalid/config.yml", code: CodeImportInvalid}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + directory := t.TempDir() + if test.setup != nil { + test.setup(t, directory) + } + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports:\n - path: \""+test.path+"\"\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) + }) + } +} + +func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) { + t.Run("cycle", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports: [{path: a.yml}]\n") + writeImportFile(t, directory, "a.yml", "_configkit:\n imports: [{path: b.yml}]\n") + writeImportFile(t, directory, "b.yml", "_configkit:\n imports: [{path: main.yml}]\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportCycle) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || len(diagnostic.Related) != 2 || !strings.Contains(diagnostic.Message, "main.yml") || !strings.Contains(diagnostic.Message, "a.yml") || !strings.Contains(diagnostic.Message, "b.yml") { + t.Fatalf("cycle diagnostic = %#v", diagnostic) + } + }) + + t.Run("diamond", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit:\n imports: [{path: left.yml}, {path: right.yml}]\n") + writeImportFile(t, directory, "left.yml", "_configkit:\n imports: [{path: shared.yml}]\n") + writeImportFile(t, directory, "right.yml", "_configkit:\n imports: [{path: shared.yml}]\n") + writeImportFile(t, directory, "shared.yml", "settings: {root: shared}\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportRepeated) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || len(diagnostic.Related) < 3 { + t.Fatalf("repeated-route diagnostic = %#v", diagnostic) + } + }) + + t.Run("symlink aliases", func(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("initial filesystem contract covers Linux and macOS") + } + directory := t.TempDir() + writeImportFile(t, directory, "target.yml", "settings: {root: target}\n") + if err := os.Symlink("target.yml", filepath.Join(directory, "first-link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if err := os.Symlink("target.yml", filepath.Join(directory, "second-link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: first-link.yml}, {path: second-link.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportRepeated) + }) +} + +func TestLoadImportsCanonicalSymlinksAndKeepsHardLinksDistinct(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("initial filesystem contract covers Linux and macOS") + } + t.Run("contained absolute symlink", func(t *testing.T) { + directory := t.TempDir() + target := writeImportFile(t, directory, "target.yml", "settings: {first: linked}\n") + if err := os.Symlink(target, filepath.Join(directory, "link.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: link.yml}]}\n") + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) + if err != nil || result.Config.Settings.First != "linked" || result.Sources()[1].Path != canonicalTestPath(t, target) { + t.Fatalf("symlink result = %#v, %v", result, err) + } + }) + + t.Run("symlink escape", func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + target := writeImportFile(t, outside, "outside.yml", "settings: {root: outside}\n") + if err := os.Symlink(target, filepath.Join(root, "escape.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, root, "main.yml", "_configkit: {imports: [{path: escape.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) + }) + + t.Run("hard links are path identities", func(t *testing.T) { + directory := t.TempDir() + one := writeImportFile(t, directory, "one.yml", "settings: {first: one}\n") + if err := os.Link(one, filepath.Join(directory, "two.yml")); err != nil { + t.Skipf("hard links unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: one.yml}, {path: two.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict) + }) + + t.Run("canonicalization failure", func(t *testing.T) { + directory := t.TempDir() + if err := os.Symlink("loop.yml", filepath.Join(directory, "loop.yml")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: loop.yml}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportInvalid) + }) +} + +func TestLoadByteAndReaderImportsRequireExplicitFilesystemContext(t *testing.T) { + directory := t.TempDir() + writeImportFile(t, directory, "fragment.yml", "settings: {first: imported}\n") + input := []byte("_configkit: {imports: [{path: fragment.yml}]}\n") + context := &FilesystemContext{BaseDir: directory, Root: directory} + for _, test := range []struct { + name string + source func() Source + }{ + {name: "bytes URI label", source: func() Source { return Bytes("s3://bucket/main.yml", input) }}, + {name: "reader URI label", source: func() Source { return Reader("https://example.invalid/main.yml", strings.NewReader(string(input))) }}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := Load[importTestConfig](test.source(), Options[importTestConfig]{Imports: true, Filesystem: context}) + if err != nil || result.Config.Settings.First != "imported" { + t.Fatalf("explicit filesystem result = %#v, %v", result, err) + } + }) + } + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true}, CodeOptionsInvalid) + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: "relative", Root: directory}}, CodeOptionsInvalid) + requireLoadCode(t, Bytes("memory.yml", input), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: directory, Root: t.TempDir()}}, CodeImportOutsideRoot) +} + +func TestFileImportRootCanContainTheSelectedFileDirectory(t *testing.T) { + root := t.TempDir() + mainPath := writeImportFile(t, root, "configs/main.yml", "_configkit: {imports: [{path: ../shared.yml}]}\n") + writeImportFile(t, root, "shared.yml", "settings: {root: shared}\n") + result, err := Load[importTestConfig](File(mainPath), Options[importTestConfig]{ + Imports: true, Filesystem: &FilesystemContext{Root: root}, + }) + if err != nil || result.Config.Settings.Root != "shared" { + t.Fatalf("explicit-root result = %#v, %v", result, err) + } + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true, Filesystem: &FilesystemContext{BaseDir: filepath.Dir(mainPath), Root: root}}, CodeOptionsInvalid) +} + +func TestImportedParseAndDecodeDiagnosticsKeepTheirSource(t *testing.T) { + t.Run("parse", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: broken.yml}]}\n") + writeImportFile(t, directory, "broken.yml", "settings: [\n") + loadError := requireAnyLoadError(t, File(mainPath), Options[importTestConfig]{Imports: true}) + diagnostics := loadError.Diagnostics() + if diagnostics[0].Code != CodeYAMLInvalid || diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "source-2" || len(loadError.Sources()) != 2 { + t.Fatalf("parse failure = diagnostics %#v, sources %#v", diagnostics, loadError.Sources()) + } + }) + + t.Run("decode", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: unknown.yml}]}\n") + writeImportFile(t, directory, "unknown.yml", "unknown: value\n") + loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeDecodeUnknownField) + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" { + t.Fatalf("decode diagnostic = %#v", diagnostic) + } + }) +} + +func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "outside.yml", "_configkit: {imports: [{path: ../missing.yml, optional: true}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) + + locked := writeImportFile(t, directory, "locked.yml", "settings: {root: locked}\n") + if err := os.Chmod(locked, 0); err != nil { + t.Skipf("cannot create unreadable fixture: %v", err) + } + defer func() { _ = os.Chmod(locked, 0o600) }() + if file, err := os.Open(locked); err == nil { + _ = file.Close() + t.Skip("filesystem does not enforce the unreadable-file fixture") + } + mainPath = writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: locked.yml, optional: true}]}\n") + requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourcePermission) +} + +func TestImportTraversalEnforcesPrivateBudgets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: one.yml}, {path: two.yml}]}\n") + writeImportFile(t, directory, "one.yml", "settings: {first: one}\n") + writeImportFile(t, directory, "two.yml", "settings: {second: two}\n") + main := parseImportMain(t, mainPath) + for _, test := range []struct { + name string + limits importLimits + }{ + {name: "source count", limits: importLimits{depth: 8, sources: 2, bytes: maxTotalBytes, nodes: maxDocumentNodes}}, + {name: "aggregate bytes", limits: importLimits{depth: 8, sources: 8, bytes: main.bytes + 1, nodes: maxDocumentNodes}}, + {name: "aggregate nodes", limits: importLimits{depth: 8, sources: 8, bytes: maxTotalBytes, nodes: countInputNodes(main.document) + 1}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, test.limits) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("budget diagnostics = %#v", diagnostics) + } + }) + } + + writeImportFile(t, directory, "one.yml", "_configkit: {imports: [{path: nested.yml}]}\n") + writeImportFile(t, directory, "nested.yml", "settings: {nested: nested}\n") + main = parseImportMain(t, mainPath) + _, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, importLimits{depth: 1, sources: 8, bytes: maxTotalBytes, nodes: maxDocumentNodes}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("depth diagnostics = %#v", diagnostics) + } +} + +func TestImportTraversalAcceptsExactPrivateBudgets(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "_configkit: {imports: [{path: imported.yml}]}\n") + importedPath := writeImportFile(t, directory, "imported.yml", "settings: {first: imported}\n") + main := parseImportMain(t, mainPath) + imported := parseImportMain(t, importedPath) + data, err := os.ReadFile(importedPath) + if err != nil { + t.Fatal(err) + } + limits := importLimits{ + depth: 1, + sources: 2, + bytes: main.bytes + int64(len(data)), + nodes: countInputNodes(main.document) + countInputNodes(imported.document), + } + composed, sources, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, limits) + if len(diagnostics) != 0 || composed == nil || len(sources) != 2 { + t.Fatalf("exact-limit composition = (%#v, %#v, %#v)", composed, sources, diagnostics) + } +} + +func writeImportFile(t *testing.T, root, relative, contents string) string { + t.Helper() + filename := filepath.Join(root, filepath.FromSlash(relative)) + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + return filename +} + +func canonicalTestPath(t *testing.T, filename string) string { + t.Helper() + canonical, err := canonicalExistingPath(filename) + if err != nil { + t.Fatal(err) + } + return canonical +} + +func requireLoadCode(t *testing.T, source Source, options Options[importTestConfig], code Code) *LoadError { + t.Helper() + loadError := requireAnyLoadError(t, source, options) + diagnostics := loadError.Diagnostics() + if len(diagnostics) == 0 || diagnostics[0].Code != code { + t.Fatalf("diagnostics = %#v, want %s", diagnostics, code) + } + return loadError +} + +func requireAnyLoadError(t *testing.T, source Source, options Options[importTestConfig]) *LoadError { + t.Helper() + result, err := Load[importTestConfig](source, options) + if result != nil || err == nil { + t.Fatalf("result, error = %#v, %v", result, err) + } + var loadError *LoadError + if !errors.As(err, &loadError) { + t.Fatalf("error type = %T", err) + } + return loadError +} + +func parseImportMain(t *testing.T, filename string) parsedSource { + t.Helper() + parsed, diagnostics := parseSource(File(filename), "source-1", defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatalf("parse diagnostics = %#v", diagnostics) + } + return parsed +} diff --git a/load.go b/load.go index 355b1ab..32fbf37 100644 --- a/load.go +++ b/load.go @@ -17,7 +17,8 @@ type Definition[T any] struct { } // Options contains the application definition and per-load input selections. -// The current single-source boundary rejects features scheduled for later work. +// Imports enables explicit path imports; glob imports and later precedence layers +// remain unavailable until their owning features are implemented. type Options[T any] struct { Definition Definition[T] Imports bool @@ -93,8 +94,9 @@ func (r *Result[T]) Diagnostics() []Diagnostic { return cloneDiagnostics(r.diagnostics) } -// Load parses and strictly decodes zero or one source. It accepts at most one -// Options value and returns no result when any error is reported. +// Load parses and strictly decodes zero or one selected source plus explicitly +// enabled imports. It accepts at most one Options value and returns no result when +// any error is reported. func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { accumulator := diagnosticAccumulator{} options, diagnostics := selectOptions(source, supplied) @@ -104,15 +106,30 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { } parsed, diagnostics := parseSource(source, "source-1", defaultParseLimits()) - if parsed.source != nil { - accumulator.sources = append(accumulator.sources, *parsed.source) - } if len(diagnostics) != 0 { + if parsed.source != nil { + accumulator.sources = append(accumulator.sources, *parsed.source) + } accumulator.add(diagnostics...) return nil, accumulator.loadError() } - if parsed.document != nil { - directiveDiagnostics := disabledDirectiveDiagnostics(parsed.document, Path{}) + + inputDocument := parsed.document + var composition *composedDocument + if options.Imports { + composition, accumulator.sources, diagnostics = loadImportedDocuments(source, options.Filesystem, parsed, options.Definition.Version, defaultImportLimits()) + if len(diagnostics) != 0 { + accumulator.add(diagnostics...) + return nil, accumulator.loadError() + } + inputDocument = composition.root + } else { + if parsed.source != nil { + accumulator.sources = append(accumulator.sources, *parsed.source) + } + } + if !options.Imports && inputDocument != nil { + directiveDiagnostics := disabledDirectiveDiagnostics(inputDocument, Path{}) if len(directiveDiagnostics) != 0 { accumulator.add(directiveDiagnostics...) return nil, accumulator.loadError() @@ -125,7 +142,7 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { return nil, accumulator.loadError() } - versionDiagnostics := checkVersion(options.Definition.Version, parsed.document) + versionDiagnostics := checkVersion(options.Definition.Version, inputDocument) accumulator.add(versionDiagnostics...) if hasErrors(versionDiagnostics) { return nil, accumulator.loadError() @@ -135,20 +152,23 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { if options.Definition.Defaults != nil { defaults = reflect.ValueOf(options.Definition.Defaults()) } - decoded, decodeIssues := decodeConfiguration(schema, defaults, parsed.document) + decoded, decodeIssues := decodeConfiguration(schema, defaults, inputDocument) if len(decodeIssues) != 0 { for _, issue := range decodeIssues { accumulator.add(diagnosticFromDecodeIssue(issue)) } return nil, accumulator.loadError() } + if composition != nil { + applyCompositionOrigins(&decoded, composition.origins) + } configuration := decoded.value.Interface().(T) metadata := metadataFromDecoded(decoded) final := decoded.final var resolutionSnapshots []resolutionSnapshot if options.Definition.Resolve != nil { - resolved := runResolution(&configuration, options.Definition.Resolve, schema, decoded, parsed.document) + resolved := runResolution(&configuration, options.Definition.Resolve, schema, decoded, inputDocument) accumulator.add(resolved.diagnostics...) if hasErrors(resolved.diagnostics) { return nil, accumulator.loadError() @@ -158,7 +178,7 @@ func Load[T any](source Source, supplied ...Options[T]) (*Result[T], error) { applyResolutionMetadata(&metadata, resolved) } if options.Definition.Validate != nil { - validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, parsed.document) + validationDiagnostics := runValidation(&configuration, options.Definition.Validate, schema, final, inputDocument) accumulator.add(validationDiagnostics...) if hasErrors(validationDiagnostics) { return nil, accumulator.loadError() @@ -187,13 +207,10 @@ func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []D if definition.Secret != nil { diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsUnavailable, "dynamic secret classification is not available in this release boundary")) } - if options.Imports { - diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsUnavailable, "configuration imports are not available in this release boundary")) - } if options.Profile != "" && options.Profile != "default" { diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsUnavailable, "configuration profiles are not available in this release boundary")) } - if options.Filesystem != nil { + if options.Filesystem != nil && !options.Imports { code := CodeOptionsUnavailable message := "filesystem context is not available in this release boundary" if source.kind == sourceNone { diff --git a/load_metadata.go b/load_metadata.go index 5c96547..895f295 100644 --- a/load_metadata.go +++ b/load_metadata.go @@ -18,14 +18,12 @@ func metadataFromDecoded(decoded decodedConfiguration) Metadata { Path: path, Origin: Origin{Layer: LayerDefault}, }) } - for pointer, input := range decoded.presence { + for pointer := range decoded.presence { path, err := ParsePath(pointer) if err != nil { continue } - origin := Origin{Layer: LayerBase, Locations: []Location{{ - SourceID: input.location.SourceID, Line: input.location.Line, Column: input.location.Column, - }}} + origin := cloneOrigin(decoded.origins[pointer]) change := Change{Path: path, Origin: origin} if _, ok := baseline[pointer]; ok { change.Previous = []Origin{{Layer: LayerDefault}} diff --git a/load_test.go b/load_test.go index 1f442c5..15af2c2 100644 --- a/load_test.go +++ b/load_test.go @@ -223,7 +223,7 @@ func TestLoadRejectsUnavailableOptionsAndDisabledDirectives(t *testing.T) { }{ {name: "multiple options", source: validSource(), options: []Options[config]{{}, {}}, code: CodeOptionsInvalid}, {name: "secret classifier", source: validSource(), options: []Options[config]{{Definition: Definition[config]{Secret: func(Path) bool { return false }}}}, code: CodeOptionsUnavailable}, - {name: "imports", source: validSource(), options: []Options[config]{{Imports: true}}, code: CodeOptionsUnavailable}, + {name: "imports without filesystem context", source: validSource(), options: []Options[config]{{Imports: true}}, code: CodeOptionsInvalid}, {name: "profile", source: validSource(), options: []Options[config]{{Profile: "testing"}}, code: CodeOptionsUnavailable}, {name: "filesystem", source: validSource(), options: []Options[config]{{Filesystem: &FilesystemContext{}}}, code: CodeOptionsUnavailable}, {name: "overrides", source: validSource(), options: []Options[config]{{Overrides: []Override{{Path: Keys("name"), Value: "other", Flag: "--name"}}}}, code: CodeOptionsUnavailable}, diff --git a/parse.go b/parse.go index a627acb..7b5e788 100644 --- a/parse.go +++ b/parse.go @@ -30,6 +30,7 @@ func defaultParseLimits() parseLimits { type parsedSource struct { document *document.Node source *SourceInfo + bytes int64 } func parseSource(source Source, sourceID string, limits parseLimits) (parsedSource, []Diagnostic) { @@ -60,7 +61,7 @@ func parseSource(source Source, sourceID string, limits parseLimits) (parsedSour }) } } - return parsedSource{source: read.info}, canonicalDiagnostics(diagnostics) + return parsedSource{source: read.info, bytes: int64(len(read.data))}, canonicalDiagnostics(diagnostics) } - return parsedSource{document: root, source: read.info}, nil + return parsedSource{document: root, source: read.info, bytes: int64(len(read.data))}, nil } diff --git a/records.go b/records.go index e4e24a7..0ddabcb 100644 --- a/records.go +++ b/records.go @@ -76,7 +76,15 @@ const ( CodeOptionsInvalid Code = "options.invalid" CodeOptionsUnavailable Code = "options.unavailable" - CodeImportDisabled Code = "import.disabled" + CodeImportDisabled Code = "import.disabled" + CodeImportInvalid Code = "import.invalid" + CodeImportUnsupported Code = "import.unsupported" + CodeImportMissing Code = "import.missing" + CodeImportOutsideRoot Code = "import.outside_root" + CodeImportCycle Code = "import.cycle" + CodeImportRepeated Code = "import.repeated" + CodeImportLimitExceeded Code = "import.limit_exceeded" + CodeImportVersion Code = "import.version" CodeComposeInvalid Code = "compose.invalid" CodeComposeConflict Code = "compose.conflict"