diff --git a/README.md b/README.md index 4235e7e..63e64e9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ConfigKit's simple `Load` accepts one strict YAML source (or explicit defaults-only input). `LoadLayered` additionally combines explicit lower-priority source groups with an independently selected main input. Sources can be host files, native `fs.FS` files, labeled bytes or readers, or snapshotted mappings. -Enabled path and glob imports compose strictly within their group; precedence +Enabled explicit imports compose strictly within their group; precedence between groups combines mappings recursively. Both entry points return a complete, validated configuration with detached metadata and safe diagnostics. Applications may perform audited defaulting and normalization through a resolver, diff --git a/docs/config.md b/docs/config.md index 03ae332..cd1bd0e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -68,7 +68,7 @@ contributes no source, and continues through defaults, decode, and projection. A missing file remains an error; it is never interpreted as no input. `Options` accepts the application definition, ordinary-source `Destination` and -`PlacementMode`, explicit path and glob imports, one selected overlay, filesystem +`PlacementMode`, explicit imports, one selected overlay, filesystem context, explicit CLI overrides, and shared source-version controls. The definition may register strict sequence composition paths through `SequenceCompositions`. Its zero-valued placement mode places the source mapping @@ -914,14 +914,18 @@ zero. Imports are opt-in. With `Options.Imports` enabled, a document may reserve its root `configkit` mapping for an `imports` sequence. ConfigKit removes that mapping before -application decoding. Each sequence item is a mapping with exactly one nonempty -string `path` or `glob`, an optional boolean `optional`, and an optional -`destination` sequence of strings. Every destination string is one literal -mapping key and follows the input-destination validation rules above; quote -numeric-looking keys so YAML reads them as strings. Null, scalar destinations, -and non-string members are `import.invalid`, as are all other unsupported shapes -and fields. A `configkit` key below the root is invalid. With imports disabled, -any occurrence remains `import.disabled`. +application decoding. Each sequence item is a mapping with exactly one `path`, +`dir`, or `glob` selector. `path` and `dir` accept a string or a list of strings; +`glob` accepts one path pattern string. The optional `required` boolean controls +whether an empty selection or a missing named path is an error. Its default is +false, so missing imports are absent unless `required: true` is specified. The +removed `optional` field is invalid. `destination` is an optional sequence of +strings. Every destination string is one literal mapping key and follows the +input-destination validation rules above; quote numeric-looking keys so YAML +reads them as strings. Null, scalar destinations, and non-string members are +`import.invalid`, as are all other unsupported shapes and fields. A `configkit` +key below the root is invalid. With imports disabled, any occurrence remains +`import.disabled`. The directive key is `configkit`, not the `_configkit` spelling used in v0.1.0. There is no legacy alias: `_configkit` is ordinary application data, subject to @@ -931,37 +935,53 @@ fields previously named `configkit` to avoid the newly reserved key. Path declarations use slash-separated relative paths. Absolute paths and network URLs are invalid. Each path resolves from its declaring file's canonical directory. -Acquisition remains depth-first in declaration order, preserving source-record and -version-callback order. Ordinary mapping contributions in a main tier are assembled -dependency-first: ConfigKit selects the lexically smallest stable source key among -the contributions whose imports are complete. Independent branches may interleave, -and the main body may be assembled after its imports. Sequence-item contributions -in that tier are nodes in the same schedule; each selected item first assembles its -own strict mapping scope, with helpers before the bodies that import them. Those -local scopes do not use application collection policies and are not flattened into -the outer schedule. Item positions and diagnostic paths follow the resulting -assembly order. -Each underlay group likewise acquires its inputs before dependency-ready assembly -in one strict scope. Independent groups remain separate precedence tiers; a higher -tier still replaces a lower tier's sequence values wholesale. -Assembly order never permits replacement. Required missing paths fail; `optional: -true` suppresses only target nonexistence, not invalid paths, containment failures, -directories, permissions, parse errors, or composition errors. +An empty `path: []` selects nothing; it succeeds even when `required: true` because +there are no named targets to be missing. A nonempty required path list verifies +every named target, including targets later excluded from selection. + +Directory declarations search the listed literal directory roots. Missing roots +contribute no candidates; supplied inaccessible or invalid roots fail. `pattern` +accepts one basename pattern or a list, matching filenames only. Omission selects +all filenames; an explicit empty list selects none. `exclude` accepts one basename +pattern or a list and removes matching final candidates. Patterns use Go's +case-sensitive `path.Match` grammar. `recursive` enables descendant traversal; +`depth` is valid only when recursion is effective, with direct files at depth 1 and +depth 0 or omission meaning unlimited traversal subject to library budgets. +`follow` controls discovered symlink entries and `hidden` controls discovered +names beginning with a dot. Both default to true. Explicit roots retain their +normal access behavior. Glob declarations use the slash-based Go `path.Match` grammar: `*`, `?`, character -classes, and backslash escapes are supported, while recursive `**` syntax is invalid. -Backslashes escape pattern characters rather than acting as directory separators. -ConfigKit validates the complete pattern before filesystem traversal, walks only -directories selected by preceding segments, and reads wildcard directories in -bounded batches. Every examined entry consumes the separate visit budget, including -nonmatches. Matches are sorted lexicographically by slash-relative spelling within -each declaration before depth-first source traversal. A required zero-match glob is -`import.missing`; `optional: true` suppresses only zero matches. Invalid patterns, -walk failures, matched directories, invalid documents, escapes, and repeated targets -remain errors. Final candidates are bounded by the remaining source limit before -their contents are accumulated. +classes, and backslash escapes are supported, while recursive `**` syntax is +invalid. Backslashes escape pattern characters rather than acting as directory +separators. ConfigKit validates the complete pattern before filesystem traversal, +walks only directories selected by preceding segments, and reads wildcard +directories in bounded batches. Every examined entry consumes the separate visit +budget, including nonmatches. Literal components are explicit path components; +`follow` and `hidden` filter only wildcard-discovered entries. `exclude` filters +final candidate basenames. -Each path or glob declaration may include `placement: mapping` or +Acquisition visits declarations in authored order and each declaration's selected +paths in namespace-path lexical order, preserving source-record and version-callback +order. Ordinary mapping contributions in a main tier are assembled dependency-first: +ConfigKit selects the lexically smallest stable source key among the contributions +whose imports are complete. Independent branches may interleave, and the main body +may be assembled after its imports. Sequence-item contributions in that tier are +nodes in the same schedule; each selected item first assembles its own strict +mapping scope, with helpers before the bodies that import them. Those local scopes +do not use application collection policies and are not flattened into the outer +schedule. Item positions and diagnostic paths follow the resulting assembly order. +Each underlay group likewise acquires its inputs before dependency-ready assembly +in one strict scope. Independent groups remain separate precedence tiers; a higher +tier still replaces a lower tier's sequence values wholesale. Assembly order never +permits replacement. + +An empty selection is allowed by default. `required: true` makes an empty directory +or glob selection fail with `import.missing`; it also makes missing named paths +fail. Requiredness does not suppress invalid paths, containment failures, +directories selected as files, permissions, parse errors, or composition errors. + +Each declaration may include `placement: mapping` or `placement: sequence-item`. Omission means `mapping` for that declaration, including every match of a glob; placement is not inherited. Sequence-item placement requires a nonempty effective destination and wraps the imported @@ -982,8 +1002,8 @@ the record's `id`. An explicit nested `sequence-item` import creates a sequence inside the record, and its placement is independent of the enclosing item. At the record root, an item import must name a nonempty nested destination. Physical paths continue to resolve from the file that declares each import. -An actual empty mapping contributes an empty item, while an optional unmatched -path or glob contributes nothing. +An actual empty mapping contributes an empty item, while an unmatched declaration +with default or explicit `required: false` contributes nothing. For host files, the configuration root defaults to the selected file's canonical directory. A host file may supply `FilesystemContext.Root` to widen that root but diff --git a/facade_source_version_mode_test.go b/facade_source_version_mode_test.go index ffb0478..0697538 100644 --- a/facade_source_version_mode_test.go +++ b/facade_source_version_mode_test.go @@ -640,7 +640,7 @@ func TestFacadeSourceVersionEvidenceOnFailedLoadsAndSyntheticSources(t *testing. Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { checked++; return nil }, }} path := filepath.Join(t.TempDir(), "main.yml") - contents := "version: " + canary + "\nconfigkit:\n imports:\n - path: missing.yml\n" + contents := "version: " + canary + "\nconfigkit:\n imports:\n - path: missing.yml\n required: true\n" if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { t.Fatal(err) } diff --git a/internal/engine/destination_accounting_test.go b/internal/engine/destination_accounting_test.go index 56243ab..e541faa 100644 --- a/internal/engine/destination_accounting_test.go +++ b/internal/engine/destination_accounting_test.go @@ -345,11 +345,11 @@ func TestAbsentOptionalImportsDoNotChargeUnusedDeepDestinations(t *testing.T) { }{ { name: "optional missing path", - body: "configkit: {imports: [{path: missing.yml, optional: true, destination: [unused]}]}\n", + body: "configkit: {imports: [{path: missing.yml, destination: [unused]}]}\n", }, { name: "optional unmatched glob", - body: "configkit: {imports: [{glob: empty/*.yml, optional: true, destination: [unused]}]}\n", + body: "configkit: {imports: [{glob: empty/*.yml, destination: [unused]}]}\n", setup: func(t *testing.T, directory string) { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { t.Fatal(err) diff --git a/internal/engine/destination_import_test.go b/internal/engine/destination_import_test.go index 2a1de96..b94e23c 100644 --- a/internal/engine/destination_import_test.go +++ b/internal/engine/destination_import_test.go @@ -451,7 +451,6 @@ func TestLayeredImportOnlyInputMaterializesDestination(t *testing.T) { mainPath := writeImportFile(t, directory, "main.yml", `configkit: imports: - path: optional.yml - optional: true `) loaded, err := LoadLayered( Input{Source: File(mainPath), Imports: true, Destination: Keys("collections", "empty")}, diff --git a/internal/engine/directives.go b/internal/engine/directives.go index 4f1a0e7..76afbcb 100644 --- a/internal/engine/directives.go +++ b/internal/engine/directives.go @@ -9,27 +9,16 @@ import ( const directiveKey = "configkit" -type importDeclaration struct { - path string - glob string - destination Path - placement placementMode - placementPath Path - placementLocation *Location - optional bool - location Location -} - -func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { +func splitImportDirectives(root *document.Node) (*document.Node, []importDeclarationSpec, []Diagnostic) { body := document.Clone(root) - var declarations []importDeclaration + var declarations []importDeclarationSpec directiveIndex := -1 for index, entry := range body.Mapping { if entry.Key != directiveKey { continue } directiveIndex = index - parsed, diagnostics := parseImportDirective(entry.Value, Keys(directiveKey)) + parsed, diagnostics := parseImportSpecificationDirective(entry.Value, Keys(directiveKey)) if len(diagnostics) != 0 { return nil, nil, diagnostics } @@ -45,101 +34,6 @@ func splitImportDirectives(root *document.Node) (*document.Node, []importDeclara return body, declarations, nil } -func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) { - if node.Kind != document.Mapping { - return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)} - } - var declarations []importDeclaration - for _, entry := range node.Mapping { - fieldPath := directivePath.Key(entry.Key) - if entry.Key != "imports" { - return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)} - } - if entry.Value.Kind != document.Sequence { - return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)} - } - for index, item := range entry.Value.Sequence { - declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index)) - if len(diagnostics) != 0 { - return nil, diagnostics - } - declarations = append(declarations, declaration) - } - } - return declarations, nil -} - -func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) { - if node.Kind != document.Mapping { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} - } - declaration := importDeclaration{location: publicLocation(node.Location), placementPath: declarationPath.clone()} - seenTarget := false - for _, entry := range node.Mapping { - fieldPath := declarationPath.Key(entry.Key) - switch entry.Key { - case "path", "glob": - if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)} - } - seenTarget = true - declaration.location = publicLocation(entry.Value.Location) - if entry.Key == "path" { - declaration.path = entry.Value.Text - } else { - declaration.glob = entry.Value.Text - } - case "optional": - if entry.Value.Kind != document.Boolean { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)} - } - declaration.optional = entry.Value.Text == "true" - case "destination": - if entry.Value.Kind != document.Sequence { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must be a sequence of strings", entry.Value)} - } - relative := Path{segments: make([]pathSegment, 0, len(entry.Value.Sequence))} - for index, item := range entry.Value.Sequence { - memberPath := fieldPath.Index(index) - if item.Kind != document.String { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be strings", item)} - } - member := Path{segments: []pathSegment{{kind: pathSegmentKey, key: item.Text}}} - if _, err := normalizeDestination(member); err != nil { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be non-reserved mapping keys", item)} - } - relative.segments = append(relative.segments, pathSegment{kind: pathSegmentKey, key: item.Text}) - } - normalized, err := normalizeDestination(relative) - if err != nil { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", entry.Value)} - } - declaration.destination = normalized - case "placement": - declaration.placementPath = fieldPath.clone() - placementLocation := publicLocation(entry.Value.Location) - declaration.placementLocation = &placementLocation - if entry.Value.Kind != document.String { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)} - } - switch entry.Value.Text { - case "mapping": - declaration.placement = placementModeMapping - case "sequence-item": - declaration.placement = placementModeSequenceItem - default: - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)} - } - default: - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} - } - } - if !seenTarget { - return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)} - } - return declaration, nil -} - func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { if node.Kind == document.Mapping { for _, entry := range node.Mapping { diff --git a/internal/engine/filesystem_links.go b/internal/engine/filesystem_links.go index f4fb1b4..5b4af6e 100644 --- a/internal/engine/filesystem_links.go +++ b/internal/engine/filesystem_links.go @@ -136,10 +136,16 @@ func resolveHostImportPath(access *filesystemAccess, name string) (string, error if err != nil { return "", err } + var targetPath string if filepath.IsAbs(target) { - return "", errImportPathOutsideRoot + relative, err := filepath.Rel(access.rootPath, filepath.Clean(target)) + if err != nil { + return "", errImportPathOutsideRoot + } + targetPath = path.Clean(filepath.ToSlash(relative)) + } else { + targetPath = path.Clean(path.Join(path.Dir(candidate), filepath.ToSlash(target))) } - targetPath := path.Clean(path.Join(path.Dir(candidate), filepath.ToSlash(target))) if !withinPortableRoot(targetPath) { return "", errImportPathOutsideRoot } diff --git a/internal/engine/glob.go b/internal/engine/glob.go index 29649e1..021fe5e 100644 --- a/internal/engine/glob.go +++ b/internal/engine/glob.go @@ -11,77 +11,6 @@ import ( const globReadBatch = 128 -type globCandidate struct { - absolute string - relative string -} - -func (s *importTraversal) expandImportGlob(basePath string, declaration importDeclaration) ([]string, []Diagnostic) { - location := declaration.location - segments, valid := validGlobPattern(declaration.glob) - if !valid { - return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid or uses recursive ** syntax", &location)} - } - - candidates := []globCandidate{{absolute: basePath}} - for index, segment := range segments { - last := index == len(segments)-1 - literal, patterned := globLiteralSegment(segment) - var next []globCandidate - for _, candidate := range candidates { - var names []string - if patterned { - matched, diagnostics := s.readGlobMatches(candidate.absolute, segment, location) - if len(diagnostics) != 0 { - return nil, diagnostics - } - names = matched - } else { - names = []string{literal} - } - for _, name := range names { - absolute := s.access.join(candidate.absolute, name) - relative := path.Join(candidate.relative, name) - if !s.access.contains(absolute) { - return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal is outside the configuration root", &location)} - } - canonical, err := s.access.canonical(absolute) - if err != nil { - if errors.Is(err, fs.ErrNotExist) && !patterned { - continue - } - return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", location)} - } - if !s.access.contains(canonical) { - return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal resolves outside the configuration root", &location)} - } - if last { - next = append(next, globCandidate{absolute: canonical, relative: relative}) - continue - } - info, err := s.access.stat(canonical) - if err != nil { - return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob traversal target could not be inspected", location)} - } - if info.IsDir() { - next = append(next, globCandidate{absolute: canonical, relative: relative}) - } - } - } - candidates = next - if len(candidates) == 0 { - return nil, nil - } - } - - matches := make([]string, len(candidates)) - for index, candidate := range candidates { - matches[index] = candidate.relative - } - slices.Sort(matches) - return matches, nil -} - // selectGlobImportCandidates expands path components while retaining the // lexical spelling used for filtering and later source admission. func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) { @@ -105,8 +34,9 @@ func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec nextByPath := make(map[string]selectedImportCandidate) for _, parent := range candidates { if !patterned { - childPath := access.join(parent.accessPath, literal) - candidate, missing, diagnostics := explicitGlobCandidate(access, childPath, site, last) + childPath := access.join(parent.path, literal) + childAccessPath := access.join(parent.accessPath, literal) + candidate, missing, diagnostics := explicitGlobCandidate(access, basePath, childPath, childAccessPath, site, last) if len(diagnostics) != 0 { return importSelection{}, diagnostics } @@ -132,10 +62,10 @@ func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec continue } accessPath := access.join(parent.accessPath, entry.Name()) - if !access.contains(accessPath) { + lexicalPath := access.join(parent.path, entry.Name()) + if !access.contains(accessPath) || !access.contains(lexicalPath) { return importSelection{}, []Diagnostic{*importSelectionDiagnostic(site, CodeImportOutsideRoot, "glob traversal is outside the configuration root")} } - candidate := selectedImportCandidate{path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: site} target, err := access.importEntryTarget(accessPath, entry, policy.follow) if err != nil { return importSelection{}, []Diagnostic{globCandidateDiagnostic(err, site.location, "glob match could not be inspected")} @@ -146,6 +76,17 @@ func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec if !last && !target.dir { continue } + physicalPath := target.identity + if physicalPath == "" { + physicalPath, err = access.canonical(accessPath) + if err != nil { + return importSelection{}, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", site.location)} + } + } + candidate := selectedImportCandidate{ + path: normalizedImportCandidatePath(access, lexicalPath), accessPath: physicalPath, + label: relativeImportLabel(access, basePath, lexicalPath), site: site, + } nextByPath[candidate.path] = candidate } } @@ -178,18 +119,18 @@ func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec return selection, nil } -func explicitGlobCandidate(access *filesystemAccess, candidatePath string, site importFieldSite, last bool) (selectedImportCandidate, bool, []Diagnostic) { - if !access.contains(candidatePath) { +func explicitGlobCandidate(access *filesystemAccess, basePath, candidatePath, accessPath string, site importFieldSite, last bool) (selectedImportCandidate, bool, []Diagnostic) { + if !access.contains(candidatePath) || !access.contains(accessPath) { return selectedImportCandidate{}, false, []Diagnostic{*importSelectionDiagnostic(site, CodeImportOutsideRoot, "glob traversal is outside the configuration root")} } - target, err := access.importEntryTarget(candidatePath, nil, true) + target, err := access.importEntryTarget(accessPath, nil, true) if err != nil { if errors.Is(err, fs.ErrNotExist) { return selectedImportCandidate{}, true, nil } return selectedImportCandidate{}, false, []Diagnostic{globCandidateDiagnostic(err, site.location, "glob match could not be inspected")} } - canonical, err := access.canonical(candidatePath) + canonical, err := access.canonical(accessPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { return selectedImportCandidate{}, true, nil @@ -205,12 +146,14 @@ func explicitGlobCandidate(access *filesystemAccess, candidatePath string, site } } return selectedImportCandidate{ - path: normalizedImportCandidatePath(access, candidatePath), accessPath: candidatePath, site: site, + path: normalizedImportCandidatePath(access, candidatePath), accessPath: canonical, + label: relativeImportLabel(access, basePath, candidatePath), site: site, }, false, nil } func readGlobEntries(access *filesystemAccess, directory string, location Location, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, []Diagnostic) { - if _, err := access.importEntryTarget(directory, nil, true); err != nil { + target, err := access.importEntryTarget(directory, nil, true) + if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, true, nil } @@ -229,8 +172,12 @@ func readGlobEntries(access *filesystemAccess, directory string, location Locati if _, err := access.relative(canonical); err != nil { return nil, false, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)} } + readPath := target.identity + if readPath == "" { + readPath = canonical + } entries := make([]fs.DirEntry, 0) - err = visitBoundedImportDirectory(access, directory, limits.entries, accounting, func(entry fs.DirEntry) error { + err = visitBoundedImportDirectory(access, readPath, limits.entries, accounting, func(entry fs.DirEntry) error { entries = append(entries, entry) return nil }) @@ -270,27 +217,6 @@ func candidatesFromMap(candidates map[string]selectedImportCandidate) []selected return result } -func (s *importTraversal) readGlobMatches(directory, pattern string, location Location) ([]string, []Diagnostic) { - entries, missing, diagnostics := readGlobEntries(s.access, directory, location, s.limits, s.accounting) - if len(diagnostics) != 0 { - return nil, diagnostics - } - if missing { - return nil, nil - } - var matches []string - for _, entry := range entries { - matched, matchErr := path.Match(pattern, entry.Name()) - if matchErr != nil { - return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)} - } - if matched { - matches = append(matches, entry.Name()) - } - } - return matches, nil -} - func validGlobPattern(pattern string) ([]string, bool) { if !validImportPath(pattern) { return nil, false diff --git a/internal/engine/glob_test.go b/internal/engine/glob_test.go index 2160c6e..0a6cff7 100644 --- a/internal/engine/glob_test.go +++ b/internal/engine/glob_test.go @@ -1,6 +1,7 @@ package engine import ( + "errors" "os" "path/filepath" "reflect" @@ -106,7 +107,7 @@ func TestLoadGlobImportsRejectInvalidAndUnmatchedPatterns(t *testing.T) { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { t.Fatal(err) } - mainPath := writeGlobRoot(t, directory, test.pattern, false) + mainPath := writeGlobRoot(t, directory, test.pattern, test.code == CodeImportMissing) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) }) } @@ -115,7 +116,7 @@ func TestLoadGlobImportsRejectInvalidAndUnmatchedPatterns(t *testing.T) { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { t.Fatal(err) } - mainPath := writeGlobRoot(t, directory, "empty/*.yml", true) + mainPath := writeGlobRoot(t, directory, "empty/*.yml", false) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) if err != nil || len(result.Sources()) != 1 { t.Fatalf("optional unmatched glob = %#v, %v", result, err) @@ -139,7 +140,7 @@ func TestLoadGlobImportsPreserveIdentityAndFileRules(t *testing.T) { if err := os.MkdirAll(filepath.Join(directory, "conf/matched.yml"), 0o755); err != nil { t.Fatal(err) } - mainPath := writeGlobRoot(t, directory, "conf/*.yml", true) + mainPath := writeGlobRoot(t, directory, "conf/*.yml", false) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourceNotRegular) }) @@ -168,6 +169,10 @@ func TestLoadGlobImportsConfineSymlinkDirectories(t *testing.T) { mainPath := writeGlobRoot(t, root, "linked/*.yml", false) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) if err != nil || result.Config.Items["linked"] != "linked" { + var loadError *LoadError + if errors.As(err, &loadError) { + t.Fatalf("contained directory symlink = %#v, diagnostics %#v", result, loadError.Diagnostics()) + } t.Fatalf("contained directory symlink = %#v, %v", result, err) } }) @@ -179,7 +184,7 @@ func TestLoadGlobImportsConfineSymlinkDirectories(t *testing.T) { if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { t.Skipf("symlinks unavailable: %v", err) } - mainPath := writeGlobRoot(t, root, "linked/*.yml", true) + mainPath := writeGlobRoot(t, root, "linked/*.yml", false) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) }) } @@ -198,7 +203,7 @@ func TestOptionalGlobDoesNotSuppressDirectoryPermissionFailure(t *testing.T) { _ = opened.Close() t.Skip("filesystem does not enforce the unreadable-directory fixture") } - mainPath := writeGlobRoot(t, directory, "locked/*.yml", true) + mainPath := writeGlobRoot(t, directory, "locked/*.yml", false) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportPermission) } @@ -207,7 +212,7 @@ func TestGlobTraversalBoundsNonmatchesAndAcceptedCandidates(t *testing.T) { for _, name := range []string{"one.txt", "two.txt", "three.txt"} { writeImportFile(t, directory, "many/"+name, "ignored\n") } - mainPath := writeGlobRoot(t, directory, "many/*.yml", true) + mainPath := writeGlobRoot(t, directory, "many/*.yml", false) main := parseImportMain(t, mainPath) limits := defaultImportLimits() limits.entries = 2 @@ -263,11 +268,11 @@ func TestFilesystemContextIsValidatedWithoutImportsOrActiveOverlay(t *testing.T) } } -func writeGlobRoot(t *testing.T, directory, pattern string, optional bool) string { +func writeGlobRoot(t *testing.T, directory, pattern string, required bool) string { t.Helper() - optionalLine := "" - if optional { - optionalLine = "\n optional: true" + requiredLine := "" + if required { + requiredLine = "\n required: true" } - return writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - glob: '"+pattern+"'"+optionalLine+"\n") + return writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - glob: '"+pattern+"'"+requiredLine+"\n") } diff --git a/internal/engine/import_selection.go b/internal/engine/import_selection.go index d86d1f9..f00c252 100644 --- a/internal/engine/import_selection.go +++ b/internal/engine/import_selection.go @@ -19,6 +19,7 @@ const ( type selectedImportCandidate struct { path string accessPath string + label string site importFieldSite } @@ -47,7 +48,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec } else { site = spec.fieldSites["path"] } - normalized = append(normalized, selectedImportCandidate{path: candidatePath, accessPath: accessPath, site: site}) + normalized = append(normalized, selectedImportCandidate{path: candidatePath, accessPath: accessPath, label: relativeImportLabel(access, basePath, accessPath), site: site}) } sort.Slice(normalized, func(left, right int) bool { return normalized[left].path < normalized[right].path @@ -111,6 +112,7 @@ func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, if targetErr != nil { return importSelection{}, []Diagnostic{*importDirectoryTargetDiagnostic(root.site, targetErr)} } + root.accessPath = rootTarget.identity entries, missing, diagnostic := importDirectoryEntries(access, root.accessPath, root.site, limits, accounting) if diagnostic != nil { return importSelection{}, []Diagnostic{*diagnostic} @@ -131,8 +133,10 @@ func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, entry := frame.entries[frame.next] frame.next++ accessPath := access.join(frame.directory.accessPath, entry.Name()) + lexicalPath := access.join(frame.directory.path, entry.Name()) candidate := selectedImportCandidate{ - path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: frame.directory.site, + path: normalizedImportCandidatePath(access, lexicalPath), accessPath: accessPath, + label: relativeImportLabel(access, basePath, lexicalPath), site: frame.directory.site, } if !policy.hidden && strings.HasPrefix(entry.Name(), ".") { continue @@ -166,11 +170,24 @@ func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, return importSelection{}, []Diagnostic{*childDiagnostic} } if !vanished { + candidate.accessPath = target.identity activeDirectories[target.identity] = struct{}{} stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries, identity: target.identity}) } continue } + if target.identity != "" { + candidate.accessPath = target.identity + } else { + canonical, canonicalErr := access.canonical(accessPath) + if canonicalErr != nil { + if errors.Is(canonicalErr, fs.ErrNotExist) { + continue + } + return importSelection{}, []Diagnostic{globFilesystemDiagnostic(canonicalErr, "directory search file could not be canonicalized", frame.directory.site.location)} + } + candidate.accessPath = canonical + } if !matchesImportFilename(entry.Name(), policy.pattern) { continue } @@ -222,8 +239,19 @@ func importDirectoryTargetDiagnostic(site importFieldSite, err error) *Diagnosti } func importDirectoryEntries(access *filesystemAccess, directory string, site importFieldSite, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, *Diagnostic) { + target, targetErr := access.importEntryTarget(directory, nil, true) + if targetErr != nil { + if errors.Is(targetErr, fs.ErrNotExist) { + return nil, true, nil + } + return nil, false, importDirectoryTargetDiagnostic(site, targetErr) + } + readPath := target.identity + if readPath == "" { + readPath = directory + } entries := make([]fs.DirEntry, 0) - err := visitBoundedImportDirectory(access, directory, limits.entries, accounting, func(entry fs.DirEntry) error { + err := visitBoundedImportDirectory(access, readPath, limits.entries, accounting, func(entry fs.DirEntry) error { entries = append(entries, entry) return nil }) @@ -255,6 +283,27 @@ func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) return filepath.ToSlash(filepath.Clean(accessPath)) } +func relativeImportLabel(access *filesystemAccess, basePath, candidatePath string) string { + if access.native != nil { + if candidatePath == basePath { + return "." + } + prefix := strings.TrimSuffix(basePath, "/") + "/" + if basePath == "." { + return candidatePath + } + if strings.HasPrefix(candidatePath, prefix) { + return strings.TrimPrefix(candidatePath, prefix) + } + return candidatePath + } + label, err := filepath.Rel(basePath, candidatePath) + if err == nil { + return filepath.ToSlash(label) + } + return filepath.ToSlash(candidatePath) +} + func matchesImportFilename(name string, patterns []string) bool { if patterns == nil { return true diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 33e0472..6b5ddfb 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -51,6 +51,7 @@ type importTraversal struct { limits importLimits rule *VersionRule versionState *sourceVersionState + defaults importDefaults scope acquisitionScope itemScope bool @@ -161,11 +162,16 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem accounting.sources++ accounting.bytes += main.bytes accounting.nodes += countInputNodes(main.document) + defaults := importDefaults{} + if versionState != nil { + defaults = snapshotImportDefaults(versionState.importDefaults) + } state := importTraversal{ access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState, - scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()}, - sources: []SourceInfo{mainInfo}, - active: make(map[resourceIdentity]int), visited: visited, + defaults: defaults, + scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()}, + sources: []SourceInfo{mainInfo}, + active: make(map[resourceIdentity]int), visited: visited, importGraph: newImportInclusionGraph(), } if mainIsItem { @@ -194,7 +200,7 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem state.visited[mainResource] = mainRoute state.stack = append(state.stack, mainRoute) var body *document.Node - var declarations []importDeclaration + var declarations []importDeclarationSpec var diagnostics []Diagnostic if processing != nil { body = processing.payload @@ -383,62 +389,62 @@ func (s *importTraversal) checkInitialLimits() *Diagnostic { return nil } -func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int, parentDestination Path) []Diagnostic { - for _, declaration := range declarations { - childDestination := appendDestination(parentDestination, declaration.destination) - if declaration.glob != "" { - matches, diagnostics := s.expandImportGlob(basePath, declaration) - if len(diagnostics) != 0 { +func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic { + policies := make([]resolvedImportPolicy, len(declarations)) + for index, spec := range declarations { + policy, diagnostics := resolveImportPolicy(spec, s.defaults) + if len(diagnostics) != 0 { + return diagnostics + } + if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 { + site := spec.fieldSites["placement"] + return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}} + } + if depth+1 > s.limits.depth { + location := spec.location + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} + } + policies[index] = policy + } + for index, spec := range declarations { + policy := policies[index] + var selection importSelection + var diagnostics []Diagnostic + switch spec.selector { + case importSelectorPath: + selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy) + case importSelectorDirectory: + selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting) + case importSelectorGlob: + selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting) + default: + location := spec.location + return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)} + } + if len(diagnostics) != 0 { + return diagnostics + } + childDestination := appendDestination(parentDestination, policy.destination) + for _, candidate := range selection.candidates { + if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 { return diagnostics } - if len(matches) == 0 { - if declaration.optional { - continue - } - location := declaration.location - return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)} - } - for _, match := range matches { - matched := declaration - matched.path = match - matched.glob = "" - if diagnostics := s.visitPath(basePath, matched, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 { - return diagnostics - } - } - continue - } - if diagnostics := s.visitPath(basePath, declaration, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 { - return diagnostics } } return nil } -func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic { - location := declaration.location - if declaration.placement == placementModeSequenceItem && len(destination.segments) == 0 { - return []Diagnostic{{ - Code: CodeImportInvalid, Severity: SeverityError, - Path: declaration.placementPath.clone(), - Message: "sequence-item placement requires a nonempty destination", - Primary: cloneLocation(declaration.placementLocation), - }} +func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic { + location := candidate.site.location + if location.SourceID == "" && location.Line == 0 && location.Column == 0 { + location = spec.location } - if depth > s.limits.depth { - return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} - } - if !validImportPath(declaration.path) { - return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)} - } - - candidate := s.access.join(basePath, declaration.path) - if !s.access.contains(candidate) { + if !s.access.contains(candidate.accessPath) { return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} } - canonical, err := s.access.canonical(candidate) + canonical, err := s.access.canonical(candidate.accessPath) if err != nil { - if declaration.optional && errors.Is(err, fs.ErrNotExist) { + if !policy.required && errors.Is(err, fs.ErrNotExist) { return nil } code := CodeImportInvalid @@ -479,13 +485,16 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati } sourceID := "source-" + strconv.Itoa(s.accounting.sources+1) kind := SourceKindFile - label := declaration.path + label := candidate.label + if label == "" { + label = candidate.path + } if s.access.native != nil { kind = SourceKindFilesystem label = s.access.label + "/" + canonical } info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} - if declaration.placement == placementModeSequenceItem { + if policy.placement == placementModeSequenceItem { info.placementMode = placementModeSequenceItem } // A source's own item placement still has an exact sequence-container @@ -496,7 +505,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati info.namespace = s.access.token } s.sources = append(s.sources, info) - data, skipped, diagnostic := s.readImport(canonical, sourceID, declaration.optional) + data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required) if diagnostic != nil { return []Diagnostic{*diagnostic} } @@ -519,7 +528,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati s.accounting.nodes += nodeCount var body *document.Node - var declarations []importDeclaration + var declarations []importDeclarationSpec var diagnostics []Diagnostic var processing *sourceVersionProcessingResult if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault { @@ -527,7 +536,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati if s.itemScope { versionDestination = appendDestination(s.itemPrefix.Index(0), destination) } - processed, failures := s.versionState.process(root, sourceVersionContribution, versionDestination, sourceID, declaration.placement) + processed, failures := s.versionState.process(root, sourceVersionContribution, versionDestination, sourceID, policy.placement) if len(failures) != 0 { return failures } @@ -541,7 +550,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati } placementDestination := destination mode := placementModeMapping - if declaration.placement == placementModeSequenceItem { + if policy.placement == placementModeSequenceItem { mode = placementModeSequenceItem } role := importRoleMapping @@ -555,7 +564,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati s.active[identity] = len(s.stack) s.visited[identity] = route s.stack = append(s.stack, route) - diagnostics := s.visitItemSource(canonical, nodeKey, declaration, destination, relativeDestination, depth, info, body, declarations, processing) + diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, processing) delete(s.active, identity) s.stack = s.stack[:len(s.stack)-1] return diagnostics @@ -604,7 +613,7 @@ func (s *importTraversal) sourceDestination(destination Path) Path { return s.itemFallback.clone() } -func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, declaration importDeclaration, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclaration, processing *sourceVersionProcessingResult) []Diagnostic { +func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic { parentScope := s.scope parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback s.scope = acquisitionScope{owner: identity} @@ -911,7 +920,7 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des } } -func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { +func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) { remaining := s.limits.bytes - s.accounting.bytes limit := min(int64(maxSourceBytes), remaining) if limit < 0 { @@ -919,7 +928,7 @@ func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]by } data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) if failure != nil { - if optional && failure.missingTarget { + if !required && failure.missingTarget { return nil, true, nil } value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") diff --git a/internal/engine/imports_test.go b/internal/engine/imports_test.go index b506709..f0479ed 100644 --- a/internal/engine/imports_test.go +++ b/internal/engine/imports_test.go @@ -227,10 +227,11 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) { {name: "unknown directive field", body: "configkit: {unknown: true}\n", code: CodeImportInvalid}, {name: "imports kind", body: "configkit: {imports: {}}\n", code: CodeImportInvalid}, {name: "declaration kind", body: "configkit: {imports: [file.yml]}\n", code: CodeImportInvalid}, - {name: "missing target", body: "configkit: {imports: [{optional: true}]}\n", code: CodeImportInvalid}, + {name: "missing target", body: "configkit: {imports: [{required: true}]}\n", code: CodeImportInvalid}, {name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, {name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, - {name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, + {name: "removed optional field", body: "configkit: {imports: [{path: a.yml, optional: true}]}\n", code: CodeImportInvalid}, + {name: "required kind", body: "configkit: {imports: [{path: a.yml, required: 'yes'}]}\n", code: CodeImportInvalid}, {name: "placement kind", body: "configkit: {imports: [{path: a.yml, placement: true}]}\n", code: CodeImportInvalid}, {name: "placement value", body: "configkit: {imports: [{path: a.yml, placement: item}]}\n", code: CodeImportInvalid}, {name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid}, @@ -473,9 +474,8 @@ func TestLoadItemGlobKeepsDefaultModeIndependentAndOptionalMissesAbsent(t *testi - glob: records/*.yml destination: [items] placement: sequence-item - - path: absent/*.yml + - glob: absent/*.yml destination: [items] - optional: true placement: sequence-item - path: other.yml destination: [other] @@ -578,7 +578,7 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { body := "configkit:\n imports:\n - path: child.yml\n" + test.destination - declarations := parseImportDeclarationsYAML(t, body) + declarations := parseImportSpecificationsYAML(t, body) if len(declarations) != 1 { t.Fatalf("declarations = %#v", declarations) } @@ -597,7 +597,7 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) { func TestParseImportPlacementDefaultsPerDeclaration(t *testing.T) { t.Parallel() - declarations := parseImportDeclarationsYAML(t, `configkit: + declarations := parseImportSpecificationsYAML(t, `configkit: imports: - path: mapping.yml - glob: records/*.yml @@ -629,7 +629,7 @@ func TestParseImportDestinationRejectsInvalidShapesAndMembers(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { body := "configkit:\n imports:\n - path: child.yml\n destination: " + test.value + "\n" - parsed, diagnostics := parseImportDeclarationYAML(body) + parsed, diagnostics := parseImportSpecificationYAML(body) if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid || diagnostics[0].Path.String() != test.wantPath { t.Fatalf("parse = %#v, want import.invalid at %q", diagnostics, test.wantPath) } @@ -652,12 +652,12 @@ func TestImportDestinationValidationPrecedesOptionalSuppression(t *testing.T) { }{ { name: "optional missing path", - body: "configkit:\n imports:\n - path: absent.yml\n optional: true\n destination: null\n", + body: "configkit:\n imports:\n - path: absent.yml\n destination: null\n", path: "/configkit/imports/0/destination", }, { name: "optional unmatched glob", - body: "configkit:\n imports:\n - glob: absent/*.yml\n optional: true\n destination: [extends]\n", + body: "configkit:\n imports:\n - glob: absent/*.yml\n destination: [extends]\n", path: "/configkit/imports/0/destination/0", }, } { @@ -678,7 +678,6 @@ func TestLoadImportMissingContainmentAndRegularFileRules(t *testing.T) { mainPath := writeImportFile(t, directory, "main.yml", `configkit: imports: - path: missing.yml - optional: true settings: {root: retained} `) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) @@ -711,12 +710,42 @@ settings: {root: retained} if test.setup != nil { test.setup(t, directory) } - mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: \""+test.path+"\"\n") + required := "" + if test.code == CodeImportMissing { + required = "\n required: true" + } + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: \""+test.path+"\""+required+"\n") requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) }) } } +func TestImportDefaultsReachOrdinaryAndGroupedDeclarations(t *testing.T) { + t.Parallel() + required := true + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: child.yml}]}\n") + writeImportFile(t, directory, "child.yml", "configkit: {imports: [{path: absent.yml}]}\n") + _, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{ + Imports: true, importDefaults: importDefaults{required: &required}, + }) + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != CodeImportMissing { + t.Fatalf("ordinary transitive required default error = %#v", err) + } + + basePath := writeImportFile(t, directory, "base.yml", "settings: {root: base}\n") + underlayPath := writeImportFile(t, directory, "underlay.yml", "configkit: {imports: [{path: absent.yml}]}\n") + _, err = loadGroupedWithSecretLookup(loadRequest[importTestConfig]{ + main: loadInput{source: File(basePath)}, + underlays: []loadGroup{{inputs: []loadInput{{source: File(underlayPath), imports: true}}}}, + importDefaults: importDefaults{required: &required}, + }, testSecretBinding(), func(string) (string, bool) { return "", false }) + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != CodeImportMissing { + t.Fatalf("grouped underlay required default error = %#v", err) + } +} + func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) { t.Run("cycle", func(t *testing.T) { directory := t.TempDir() @@ -1073,7 +1102,7 @@ func TestImportedParseAndDecodeDiagnosticsKeepTheirSource(t *testing.T) { func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) { directory := t.TempDir() - mainPath := writeImportFile(t, directory, "outside.yml", "configkit: {imports: [{path: ../missing.yml, optional: true}]}\n") + mainPath := writeImportFile(t, directory, "outside.yml", "configkit: {imports: [{path: ../missing.yml}]}\n") requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) locked := writeImportFile(t, directory, "locked.yml", "settings: {root: locked}\n") @@ -1085,7 +1114,7 @@ func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) { _ = file.Close() t.Skip("filesystem does not enforce the unreadable-file fixture") } - mainPath = writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: locked.yml, optional: true}]}\n") + mainPath = writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: locked.yml}]}\n") requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourcePermission) } @@ -1195,16 +1224,16 @@ func parseImportMain(t *testing.T, filename string) parsedSource { return parsed } -func parseImportDeclarationsYAML(t *testing.T, input string) []importDeclaration { +func parseImportSpecificationsYAML(t *testing.T, input string) []importDeclarationSpec { t.Helper() - declarations, diagnostics := parseImportDeclarationYAML(input) + declarations, diagnostics := parseImportSpecificationYAML(input) if len(diagnostics) != 0 { t.Fatalf("import parse diagnostics = %#v", diagnostics) } return declarations } -func parseImportDeclarationYAML(input string) ([]importDeclaration, []Diagnostic) { +func parseImportSpecificationYAML(input string) ([]importDeclarationSpec, []Diagnostic) { parsed, diagnostics := parseSource(Bytes("import fixture", []byte(input)), "source-1", defaultParseLimits()) if len(diagnostics) != 0 { return nil, diagnostics diff --git a/internal/engine/layered.go b/internal/engine/layered.go index 8b0d732..a75a850 100644 --- a/internal/engine/layered.go +++ b/internal/engine/layered.go @@ -17,6 +17,7 @@ type Underlay struct{ Inputs []Input } // LayeredOptions selects lower-priority tiers and final application rules. type LayeredOptions[T any] struct { Definition Definition[T] + importDefaults importDefaults SourceVersionMode sourceVersionMode Underlays []Underlay Overlay string @@ -40,6 +41,7 @@ func LoadLayered[T any](main Input, secrets SecretBinding, supplied ...LayeredOp destination: main.Destination.clone(), placementMode: main.placementMode, }, definition: options.Definition, sourceVersionMode: options.SourceVersionMode, + importDefaults: snapshotImportDefaults(options.importDefaults), overlay: options.Overlay, overlayFilesystem: options.OverlayFilesystem, overrides: append([]Override(nil), options.Overrides...), diff --git a/internal/engine/load.go b/internal/engine/load.go index e564685..fa571fd 100644 --- a/internal/engine/load.go +++ b/internal/engine/load.go @@ -21,7 +21,7 @@ type Definition[T any] struct { } // Options contains the application definition and per-load input selections. -// Imports enables explicit path and glob imports; Overlay selects one fixed overlay; +// Imports enables explicit import declarations; Overlay selects one fixed overlay; // Overrides contains only values explicitly supplied by an application CLI. type Options[T any] struct { Definition Definition[T] @@ -153,7 +153,7 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se definition.sequenceRegistrations = nil request := loadRequest[T]{ main: loadInput{source: source, imports: options.Imports, filesystem: options.Filesystem, destination: options.destination.clone(), placementMode: options.placementMode}, - definition: definition, sequencePolicy: policy, overlay: options.Overlay, + definition: definition, importDefaults: snapshotImportDefaults(options.importDefaults), sequencePolicy: policy, overlay: options.Overlay, overrides: options.Overrides, ordinary: true, sourceVersionMode: options.SourceVersionMode, effectiveVersion: effectiveSourceVersionRuleForPlacement(options.Definition.Version, options.destination, options.SourceVersionMode, options.placementMode), @@ -165,6 +165,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo accumulator := diagnosticAccumulator{} accounting := &importAccounting{} mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode} + mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults) mainOptions.sequencePolicy = request.sequencePolicy mainOptions.SourceVersionMode = request.sourceVersionMode mainOptions.Definition.Version = request.protectionVersion() @@ -184,6 +185,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo } versionState := &sourceVersionState{ localRule: request.definition.Version, effectiveRule: request.protectionVersion(), mode: request.sourceVersionMode, + importDefaults: snapshotImportDefaults(request.importDefaults), } accumulator.versionState = versionState input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState) diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index df1a12e..74486b4 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -10,6 +10,7 @@ type loadRequest[T any] struct { main loadInput underlays []loadGroup definition Definition[T] + importDefaults importDefaults sequencePolicy sequencePolicy sourceVersionMode sourceVersionMode effectiveVersion *VersionRule @@ -39,13 +40,15 @@ type loadGroup struct{ inputs []loadInput } func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) { request.ordinary = false var diagnostics []Diagnostic + request.importDefaults = snapshotImportDefaults(request.importDefaults) + diagnostics = append(diagnostics, validateImportDefaults(request.importDefaults)...) policy, policyDiagnostics := compileSequencePolicy(request.definition.sequenceRegistrations) request.sequencePolicy = policy diagnostics = append(diagnostics, policyDiagnostics...) request.definition.sequenceRegistrations = nil mainOptions, mainDiagnostics := selectOptions(request.main.source, []Options[T]{{ Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), - SourceVersionMode: request.sourceVersionMode, placementMode: request.main.placementMode, + SourceVersionMode: request.sourceVersionMode, placementMode: request.main.placementMode, importDefaults: request.importDefaults, }}) request.main.destination = mainOptions.destination diagnostics = append(diagnostics, mainDiagnostics...) @@ -53,7 +56,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi for groupIndex := range request.underlays { for inputIndex := range request.underlays[groupIndex].inputs { input := &request.underlays[groupIndex].inputs[inputIndex] - inputOptions, inputDiagnostics := selectOptions(input.source, []Options[T]{{Imports: input.imports, Filesystem: input.filesystem, destination: input.destination.clone(), placementMode: input.placementMode}}) + inputOptions, inputDiagnostics := selectOptions(input.source, []Options[T]{{Imports: input.imports, Filesystem: input.filesystem, destination: input.destination.clone(), placementMode: input.placementMode, importDefaults: request.importDefaults}}) input.destination = inputOptions.destination diagnostics = append(diagnostics, inputDiagnostics...) } @@ -175,6 +178,12 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}, destination: contribution.destination.clone(), placementMode: contribution.placementMode, sequencePolicy: policy, deferAssembly: true, + importDefaults: func() importDefaults { + if versionState == nil { + return importDefaults{} + } + return snapshotImportDefaults(versionState.importDefaults) + }(), } acquired, diagnostics := assembleBaseInputTrackedForRole(contribution.source, options, contribution.destination, accounting, visited, hostToken, versionState, sourceVersionContribution) for _, source := range acquired.sources { diff --git a/internal/engine/native_filesystem_test.go b/internal/engine/native_filesystem_test.go index 6df5f8e..d6e6cf4 100644 --- a/internal/engine/native_filesystem_test.go +++ b/internal/engine/native_filesystem_test.go @@ -109,7 +109,7 @@ func TestNativeFilesystemNamespaceAndPathFailures(t *testing.T) { t.Errorf("%q diagnostics = %#v", name, diagnostics) } } - missingImport := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: host-only.yml}]}\n")}} + missingImport := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: host-only.yml, required: true}]}\n")}} _, err := testLoad[importTestConfig](filesystemFile(newNativeNamespace("bundle", missingImport), "main.yml"), Options[importTestConfig]{Imports: true}) if err == nil { t.Fatal("missing virtual import fell back to the host filesystem") @@ -170,9 +170,7 @@ func TestNativeFilesystemRootAndOptionalImports(t *testing.T) { imports: - path: ../shared.yml - path: absent.yml - optional: true - glob: absent/*.yml - optional: true `)}, "config/shared.yml": &fstest.MapFile{Data: []byte("items: {shared: retained}\n")}, "outside.yml": &fstest.MapFile{Data: []byte("items: {outside: blocked}\n")}, diff --git a/internal/engine/source_version.go b/internal/engine/source_version.go index f7ce2bc..238a1a6 100644 --- a/internal/engine/source_version.go +++ b/internal/engine/source_version.go @@ -47,6 +47,7 @@ type sourceVersionState struct { localRule *VersionRule effectiveRule *VersionRule mode sourceVersionMode + importDefaults importDefaults mainProcessing *sourceVersionProcessingResult evidence []*sourceVersionEvidence diagnostics []Diagnostic @@ -136,7 +137,7 @@ func (state *sourceVersionState) process(root *document.Node, role sourceVersion type sourceVersionProcessingResult struct { payload *document.Node - imports []importDeclaration + imports []importDeclarationSpec evidence *sourceVersionEvidence diagnostics []Diagnostic } diff --git a/internal/engine/source_version_test.go b/internal/engine/source_version_test.go index b1fd252..dde6f5f 100644 --- a/internal/engine/source_version_test.go +++ b/internal/engine/source_version_test.go @@ -880,7 +880,6 @@ func TestSourceVersionImportsRejectHiddenAndMissingDeclarations(t *testing.T) { configkit: imports: - path: missing.yml - optional: true `) checks := 0 loaded, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{ diff --git a/lifecycle.go b/lifecycle.go index 4bded67..e9183f3 100644 --- a/lifecycle.go +++ b/lifecycle.go @@ -73,7 +73,7 @@ const ( ) // Options contains the application definition and per-load input selections. -// Imports enables explicit path and glob imports; Overlay selects one fixed overlay; +// Imports enables explicit import declarations; Overlay selects one fixed overlay; // Overrides contains only values explicitly supplied by an application CLI. // Enabled source-version modes interpret Definition.Version.Path in each // document's local coordinates. Placement projects checks and main protection