Provide passive per-source resource contexts

This commit is contained in:
2026-09-29 18:08:27 +00:00
parent 5a77728a81
commit 36362ee6af
16 changed files with 752 additions and 13 deletions

View File

@@ -78,6 +78,7 @@ type Loaded[T any] struct {
type Inspection struct {
metadata Metadata
sources []SourceInfo
contexts map[string]ResourceContext
diagnostics []Diagnostic
baseline *document.Node
input *document.Node
@@ -105,6 +106,16 @@ func (r *Inspection) Sources() []SourceInfo {
return cloneSourceInfos(r.sources)
}
// SourceContext returns a passive filesystem view for an acquired source.
// Unknown IDs and sources without a filesystem context return false.
func (r *Inspection) SourceContext(id string) (ResourceContext, bool) {
if r == nil {
return ResourceContext{}, false
}
context, ok := r.contexts[id]
return context, ok
}
// Diagnostics returns detached nonfatal diagnostics from the load.
func (r *Inspection) Diagnostics() []Diagnostic {
if r == nil {
@@ -167,6 +178,22 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
if len(diagnostics) != 0 {
return nil, accumulator.loadError()
}
if request.ordinary && input.parsed.source != nil {
baseSources := make([]SourceInfo, 0, len(input.sources))
for _, source := range input.sources {
if source.Layer == LayerBase {
baseSources = append(baseSources, source)
}
}
if len(baseSources) != 0 && input.contexts[baseSources[0].ID].FS == nil {
contexts, diagnostic := contextsForAcquiredSources(request.main.source, request.main.filesystem, input.parsed.source, baseSources)
if diagnostic != nil {
accumulator.add(*diagnostic)
return nil, accumulator.loadError()
}
mergeResourceContexts(&input.contexts, contexts)
}
}
decoded, diagnostics := decodeLoadInput(schema, request.definition, input)
accumulator.add(diagnostics...)

View File

@@ -12,6 +12,7 @@ type assembledInput struct {
document *document.Node
composition *composedDocument
sources []SourceInfo
contexts map[string]ResourceContext
}
func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
@@ -56,6 +57,13 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], accounti
accounting.bytes += parsed.bytes
accounting.nodes += countInputNodes(input.document)
}
if !activeOverlay(options.Overlay) || options.Imports {
contexts, diagnostic := contextsForAcquiredSources(source, options.Filesystem, parsed.source, input.sources)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
input.contexts = contexts
}
return input, nil
}
@@ -94,6 +102,11 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI
if len(diagnostics) != 0 {
return input, diagnostics
}
contexts, diagnostic := contextsForAcquiredSources(source, options.Filesystem, input.parsed.source, input.sources[len(input.sources)-1:])
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
mergeResourceContexts(&input.contexts, contexts)
}
if len(options.Overrides) != 0 {
input.sources, diagnostics = assembly.applyOverrides(

View File

@@ -12,9 +12,10 @@ import (
type resolutionInput struct {
metadata Metadata
sources map[string]SourceInfo
contexts map[string]ResourceContext
}
func newResolutionInput(metadata Metadata, final *document.Node, sources []SourceInfo) resolutionInput {
func newResolutionInput(metadata Metadata, final *document.Node, sources []SourceInfo, contexts ...map[string]ResourceContext) resolutionInput {
frozen := Metadata{
fields: make(map[string]FieldInfo, len(metadata.fields)),
effectivePaths: effectivePathIndex(final),
@@ -32,7 +33,11 @@ func newResolutionInput(metadata Metadata, final *document.Node, sources []Sourc
}
indexed[source.ID] = cloneSourceInfo(source)
}
return resolutionInput{metadata: frozen, sources: indexed}
result := resolutionInput{metadata: frozen, sources: indexed}
if len(contexts) != 0 {
result.contexts = cloneResourceContexts(contexts[0])
}
return result
}
func (input resolutionInput) lookup(path Path) (FieldInfo, bool) {
@@ -47,6 +52,11 @@ func (input resolutionInput) source(id string) (SourceInfo, bool) {
return cloneSourceInfo(source), true
}
func (input resolutionInput) sourceContext(id string) (ResourceContext, bool) {
context, ok := input.contexts[id]
return context, ok
}
func metadataFromDecoded(decoded decodedConfiguration) Metadata {
metadata := Metadata{
fields: make(map[string]FieldInfo),

View File

@@ -56,8 +56,9 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
groups := make([]*composedDocument, 0, len(request.underlays))
for index, group := range request.underlays {
composition, sources, diagnostics := acquireUnderlayGroup(group, index+1, request.definition.Version, accounting)
composition, sources, contexts, diagnostics := acquireUnderlayGroup(group, index+1, request.definition.Version, accounting)
input.sources = append(input.sources, sources...)
mergeResourceContexts(&input.contexts, contexts)
if len(diagnostics) != 0 {
return input, diagnostics
}
@@ -106,6 +107,11 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
if len(diagnostics) != 0 {
return input, diagnostics
}
contexts, diagnostic := contextsForAcquiredSources(selection, request.overlayFilesystem, selected.source, input.sources[len(input.sources)-1:])
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
mergeResourceContexts(&input.contexts, contexts)
}
if len(request.overrides) != 0 {
var diagnostics []Diagnostic
@@ -119,11 +125,12 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
return input, nil
}
func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
visited := make(map[resourceIdentity]importRoute)
hostToken := &namespaceToken{}
var documents []*document.Node
var sources []SourceInfo
var contexts map[string]ResourceContext
for _, contribution := range group.inputs {
options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}}
acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, accounting, visited, hostToken)
@@ -132,15 +139,16 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
source.UnderlayIndex = index
sources = append(sources, source)
}
mergeResourceContexts(&contexts, acquired.contexts)
if len(diagnostics) != 0 {
return nil, sources, diagnostics
return nil, sources, contexts, diagnostics
}
if acquired.document == nil {
continue
}
if node := fragmentVersionClaim(acquired.document, rule); node != nil {
location := publicLocation(node.Location)
return nil, sources, []Diagnostic{{
return nil, sources, contexts, []Diagnostic{{
Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location,
}}
@@ -154,11 +162,11 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
}
}
if len(documents) == 0 {
return nil, sources, nil
return nil, sources, contexts, nil
}
composition, diagnostics := composeDocuments(documents...)
if len(diagnostics) != 0 {
return nil, sources, diagnostics
return nil, sources, contexts, diagnostics
}
for pointer, origin := range composition.origins {
origin.Layer = LayerUnderlay
@@ -167,5 +175,5 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
for index := range composition.applications {
composition.applications[index].layer = LayerUnderlay
}
return composition, sources, nil
return composition, sources, contexts, nil
}

View File

@@ -169,13 +169,13 @@ func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) {
accounting := &importAccounting{sources: maxImportSources - 1}
group := loadGroup{inputs: []loadInput{{source: Mapping("first", map[string]any{})}, {source: Mapping("second", map[string]any{})}}}
_, _, diagnostics := acquireUnderlayGroup(group, 1, nil, accounting)
_, _, _, diagnostics := acquireUnderlayGroup(group, 1, nil, accounting)
if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
t.Fatalf("shared source budget = %#v", diagnostics)
}
accounting = &importAccounting{bytes: maxTotalBytes - 2}
group.inputs = []loadInput{{source: Mapping("first", map[string]any{"port": "one"})}}
_, _, diagnostics = acquireUnderlayGroup(group, 1, nil, accounting)
_, _, _, diagnostics = acquireUnderlayGroup(group, 1, nil, accounting)
if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
t.Fatalf("shared byte budget = %#v", diagnostics)
}

View File

@@ -15,6 +15,7 @@ type resolvedLoad[T any] struct {
final *document.Node
snapshots []resolutionSnapshot
applications []historyApplication
contexts map[string]ResourceContext
}
func decodeLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput) (decodedConfiguration, []Diagnostic) {
@@ -51,7 +52,7 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
final := decoded.final
var resolutionSnapshots []resolutionSnapshot
if definition.Resolve != nil {
entry := newResolutionInput(metadata, decoded.final, input.sources)
entry := newResolutionInput(metadata, decoded.final, input.sources, input.contexts)
resolved := runResolution(&configuration, definition.Resolve, schema, decoded, input.document, entry)
diagnostics = append(diagnostics, resolved.diagnostics...)
if hasErrors(resolved.diagnostics) {
@@ -67,7 +68,7 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
} else if input.document != nil {
applications = []historyApplication{{layer: LayerBase, value: input.document}}
}
return resolvedLoad[T]{configuration: configuration, decoded: decoded, metadata: metadata, final: final, snapshots: resolutionSnapshots, applications: applications}, diagnostics
return resolvedLoad[T]{configuration: configuration, decoded: decoded, metadata: metadata, final: final, snapshots: resolutionSnapshots, applications: applications, contexts: input.contexts}, diagnostics
}
func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, mode secretCloneMode) (T, *Diagnostic) {
@@ -95,6 +96,7 @@ func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T]
inspection := &Inspection{
metadata: metadata,
sources: cloneSourceInfos(accumulator.sources),
contexts: cloneResourceContexts(resolved.contexts),
diagnostics: canonicalDiagnostics(accumulator.diagnostics),
baseline: decoded.baseline,
input: decoded.final,

View File

@@ -72,6 +72,12 @@ func (r *Resolution) Source(id string) (SourceInfo, bool) {
return r.activeState().input.source(id)
}
// SourceContext returns a passive context during the active resolution
// callback. Retaining the callback context after return is unsupported.
func (r *Resolution) SourceContext(id string) (ResourceContext, bool) {
return r.activeState().input.sourceContext(id)
}
// Record declares the intent for a resolver mutation. A container path covers
// mutations to its descendants.
func (r *Resolution) Record(path Path, kind ChangeKind, reason string, dependencies ...Path) {

View File

@@ -0,0 +1,132 @@
package engine
import (
"errors"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
)
// ResourceContext is a passive copy of one acquired source's rooted filesystem
// and slash-relative base directory. FS is borrowed and opens resources only
// when the application explicitly requests them.
type ResourceContext struct {
FS fs.FS
BaseDir string
}
func cloneResourceContexts(contexts map[string]ResourceContext) map[string]ResourceContext {
if contexts == nil {
return nil
}
cloned := make(map[string]ResourceContext, len(contexts))
for id, context := range contexts {
cloned[id] = context
}
return cloned
}
// rootedNativeFS borrows a namespace without invoking SubFS or opening it at
// query time. All resource names remain relative to its configured root.
type rootedNativeFS struct {
filesystem fs.FS
root string
}
func (r rootedNativeFS) name(name string) (string, error) {
if !fs.ValidPath(name) || strings.Contains(name, "\\") {
return "", &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
}
joined := path.Join(r.root, name)
if !fs.ValidPath(joined) || r.root != "." && joined != r.root && !strings.HasPrefix(joined, r.root+"/") {
return "", &fs.PathError{Op: "open", Path: name, Err: fs.ErrPermission}
}
return joined, nil
}
func (r rootedNativeFS) Open(name string) (fs.File, error) {
joined, err := r.name(name)
if err != nil {
return nil, err
}
return r.filesystem.Open(joined)
}
func (r rootedNativeFS) ReadDir(name string) ([]fs.DirEntry, error) {
joined, err := r.name(name)
if err != nil {
return nil, err
}
return fs.ReadDir(r.filesystem, joined)
}
// rootedHostFS stores only a canonical root path. No os.Root is retained after
// loading or created by a SourceContext query.
type hostResourceRoot interface {
Open(string) (*os.File, error)
Close() error
}
type rootedHostFS struct {
root string
openRoot func(string) (hostResourceRoot, error)
}
func (r rootedHostFS) Open(name string) (fs.File, error) {
if !fs.ValidPath(name) || strings.Contains(name, "\\") {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
}
opener := r.openRoot
if opener == nil {
opener = func(name string) (hostResourceRoot, error) { return os.OpenRoot(name) }
}
root, err := opener(r.root)
if err != nil {
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
}
canonical, err := canonicalExistingPath(filepath.Join(r.root, filepath.FromSlash(name)))
if err != nil {
_ = root.Close()
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
}
if !containedPath(r.root, canonical) {
_ = root.Close()
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrPermission}
}
relative, err := filepath.Rel(r.root, canonical)
if err != nil {
_ = root.Close()
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
}
file, err := root.Open(relative)
if err != nil {
_ = root.Close()
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
}
owned := rootedHostFile{File: file, root: root}
var opened fs.File = file
if directory, ok := opened.(fs.ReadDirFile); ok {
return &rootedHostDirFile{rootedHostFile: owned, directory: directory}, nil
}
return &owned, nil
}
type rootedHostFile struct {
fs.File
root hostResourceRoot
}
func (f *rootedHostFile) Close() error {
return errors.Join(f.File.Close(), f.root.Close())
}
type rootedHostDirFile struct {
rootedHostFile
directory fs.ReadDirFile
}
func (f *rootedHostDirFile) ReadDir(count int) ([]fs.DirEntry, error) {
return f.directory.ReadDir(count)
}

View File

@@ -0,0 +1,133 @@
package engine
import (
"errors"
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
)
type trackedHostRoot struct {
root *os.Root
closed *int
failOpen bool
}
func (r *trackedHostRoot) Open(name string) (*os.File, error) {
if r.failOpen {
return nil, fs.ErrPermission
}
return r.root.Open(name)
}
func (r *trackedHostRoot) Close() error {
(*r.closed)++
return r.root.Close()
}
func TestRootedHostFSClosesRootOnSuccessfulAndFailedOpen(t *testing.T) {
rootPath := t.TempDir()
if err := os.WriteFile(filepath.Join(rootPath, "resource.txt"), []byte("resource"), 0o600); err != nil {
t.Fatal(err)
}
opened, closed := 0, 0
failOpen := false
filesystem := rootedHostFS{root: rootPath, openRoot: func(name string) (hostResourceRoot, error) {
root, err := os.OpenRoot(name)
if err != nil {
return nil, err
}
opened++
return &trackedHostRoot{root: root, closed: &closed, failOpen: failOpen}, nil
}}
if _, err := filesystem.Open("../resource.txt"); !errors.Is(err, fs.ErrInvalid) || opened != 0 {
t.Fatalf("invalid path = %v; roots opened %d", err, opened)
}
if _, err := filesystem.Open("missing.txt"); !errors.Is(err, fs.ErrNotExist) || opened != 1 || closed != 1 {
t.Fatalf("missing resource = %v; opened %d, closed %d", err, opened, closed)
}
failOpen = true
if _, err := filesystem.Open("resource.txt"); !errors.Is(err, fs.ErrPermission) || opened != 2 || closed != 2 {
t.Fatalf("failed inner open = %v; opened %d, closed %d", err, opened, closed)
}
failOpen = false
file, err := filesystem.Open("resource.txt")
if err != nil || opened != 3 || closed != 2 {
t.Fatalf("resource open = %v; opened %d, closed %d", err, opened, closed)
}
if err := file.Close(); err != nil || closed != 3 {
t.Fatalf("resource close = %v; opened %d, closed %d", err, opened, closed)
}
if _, err := file.Read(make([]byte, 1)); err == nil {
t.Fatal("closed resource file remained readable")
}
directory, err := filesystem.Open(".")
if err != nil {
t.Fatal(err)
}
reader, ok := directory.(fs.ReadDirFile)
if !ok {
t.Fatal("rooted host directory did not expose ReadDirFile")
}
entries, err := reader.ReadDir(-1)
if err != nil || len(entries) != 1 || entries[0].Name() != "resource.txt" {
t.Fatalf("directory entries = %#v, %v", entries, err)
}
if err := directory.Close(); err != nil || opened != 4 || closed != 4 {
t.Fatalf("directory close = %v; opened %d, closed %d", err, opened, closed)
}
}
func TestNativeResourceViewIsBorrowedAndRooted(t *testing.T) {
filesystem := &observedFilesystem{files: fstest.MapFS{
"bundle/config/resource.txt": &fstest.MapFile{Data: []byte("content")},
}}
view := rootedNativeFS{filesystem: filesystem, root: "bundle"}
if filesystem.opened != 0 {
t.Fatal("constructing a native resource view opened files")
}
data, err := fs.ReadFile(view, "config/resource.txt")
if err != nil || string(data) != "content" {
t.Fatalf("resource = %q, %v", data, err)
}
filesystem.assertClosed(t)
if _, err := fs.ReadFile(view, "../outside.txt"); !errors.Is(err, fs.ErrInvalid) {
t.Fatalf("resource escape = %v", err)
}
}
func TestLoadingClosesNativeFilesAcrossPipelineOutcomes(t *testing.T) {
type config struct {
Value string `yaml:"value"`
Version string `yaml:"version"`
}
for _, test := range []struct {
name string
main string
child string
failRead string
options Options[config]
wantErr bool
}{
{name: "success", main: "value: retained\n"},
{name: "acquisition", main: "value: retained\n", failRead: "main.yml", wantErr: true},
{name: "parse", main: "value: [broken\n", wantErr: true},
{name: "composition", main: "configkit: {imports: [{path: child.yml}]}\nvalue: first\n", child: "value: second\n", options: Options[config]{Imports: true}, wantErr: true},
{name: "version", main: "version: wrong\n", options: Options[config]{Definition: Definition[config]{Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return fs.ErrInvalid }}}}, wantErr: true},
{name: "validation", main: "value: retained\n", options: Options[config]{Definition: Definition[config]{Validate: func(_ *config, validation *Validation) { validation.Error(Keys("value"), "rejected") }}}, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
files := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte(test.main)}}
if test.child != "" {
files["child.yml"] = &fstest.MapFile{Data: []byte(test.child)}
}
filesystem := &observedFilesystem{files: files, failRead: test.failRead}
_, err := testLoad(filesystemFile(newNativeNamespace("bundle", filesystem), "main.yml"), test.options)
if (err != nil) != test.wantErr {
t.Fatalf("load error = %v, want failure %v", err, test.wantErr)
}
filesystem.assertClosed(t)
})
}
}

View File

@@ -0,0 +1,112 @@
package engine
import (
"io/fs"
"path"
"path/filepath"
"strings"
)
func contextsForAcquiredSources(source Source, context *FilesystemContext, baseSource *SourceInfo, sources []SourceInfo) (map[string]ResourceContext, *Diagnostic) {
if len(sources) == 0 {
return nil, nil
}
var rootPath, basePath string
var filesystem fs.FS
if source.kind == sourceFilesystem {
filesystem = source.namespace.filesystem
rootPath, basePath = ".", path.Dir(source.path)
if context != nil && context.Root != "" {
rootPath = context.Root
}
} else if context != nil && context.FS != nil {
filesystem = context.FS.namespace.filesystem
rootPath, basePath = context.Root, context.BaseDir
} else if source.kind == sourceFile {
filePath := sources[0].Path
if baseSource != nil {
filePath = baseSource.Path
}
basePath = filepath.Dir(filePath)
rootPath = basePath
if context != nil && context.Root != "" {
canonical, err := canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return nil, &value
}
rootPath = canonical
}
} else if context != nil {
var err error
rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return nil, &value
}
basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return nil, &value
}
} else {
return nil, nil
}
var rooted fs.FS
if filesystem != nil {
rooted = rootedNativeFS{filesystem: filesystem, root: rootPath}
} else {
rooted = rootedHostFS{root: rootPath}
}
result := make(map[string]ResourceContext, len(sources))
for _, acquired := range sources {
base := basePath
if acquired.Path != "" {
if filesystem != nil {
base = path.Dir(acquired.Path)
} else {
base = filepath.Dir(acquired.Path)
}
}
var relative string
if filesystem != nil {
if !fs.ValidPath(rootPath) || !fs.ValidPath(base) || rootPath != "." && base != rootPath && !strings.HasPrefix(base, rootPath+"/") {
value := importDiagnostic(CodeImportOutsideRoot, "source base is outside the configuration root", nil)
return nil, &value
}
relative = base
if rootPath != "." {
relative = strings.TrimPrefix(strings.TrimPrefix(base, rootPath), "/")
if relative == "" {
relative = "."
}
}
} else {
if !containedPath(rootPath, base) {
value := importDiagnostic(CodeImportOutsideRoot, "source base is outside the configuration root", nil)
return nil, &value
}
var err error
relative, err = filepath.Rel(rootPath, base)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "source base could not be addressed in the configuration root", nil)
return nil, &value
}
relative = filepath.ToSlash(relative)
}
result[acquired.ID] = ResourceContext{FS: rooted, BaseDir: relative}
}
return result, nil
}
func mergeResourceContexts(target *map[string]ResourceContext, incoming map[string]ResourceContext) {
if len(incoming) == 0 {
return
}
if *target == nil {
*target = make(map[string]ResourceContext, len(incoming))
}
for id, context := range incoming {
(*target)[id] = context
}
}