27 Commits

Author SHA1 Message Date
d898cb193d Record the remaining context consolidation gap 2026-10-25 14:00:29 +00:00
557e006a4a Complete consolidation roadmap validation 2026-10-25 13:51:36 +00:00
3fd83b6b2b Document consumer and internal loading flows 2026-10-25 13:49:44 +00:00
0fbce5d141 Separate schema field discovery and assembly 2026-10-25 13:46:51 +00:00
ac4ce242a2 Separate reflection projection handlers 2026-10-25 13:45:27 +00:00
fa6e876ab5 Extract inheritance accounting and finalization 2026-10-25 13:43:26 +00:00
c5275df1fc Return inheritance combination evidence together 2026-10-25 13:41:41 +00:00
f404168900 Extract inheritance input preparation 2026-10-25 13:32:01 +00:00
729c6fb5d5 Extract imported source admission from traversal 2026-10-25 13:28:43 +00:00
d314a269d5 Separate directory entry admission from traversal 2026-10-25 13:25:37 +00:00
2c3eb3308b Extract import declaration parsing helpers 2026-10-25 13:23:09 +00:00
6bc625b68f Split import acquisition and projection responsibilities 2026-10-25 13:21:29 +00:00
b915dd776a Consolidate selected overlay acquisition 2026-10-25 13:18:54 +00:00
9a64a65b6c Carry resolved filesystem facts through loads 2026-10-25 13:13:47 +00:00
6158e9db3b Separate filesystem context from access 2026-10-25 13:09:44 +00:00
0d149e51f0 Wire load lifecycle through input acquisition 2026-10-25 13:07:16 +00:00
bd51b9c3f2 Consolidate import request ownership 2026-10-25 13:02:46 +00:00
d05426d8a2 Introduce non-generic input acquisition requests 2026-10-25 12:55:04 +00:00
3f0efcb4ff Normalize ordinary and grouped load preflight 2026-10-25 12:52:50 +00:00
19f5c2a2d5 Remove unused reverse source conversions 2026-10-25 12:50:11 +00:00
455a4a355d Map ConfigKit consolidation callers and coverage 2026-10-25 12:37:33 +00:00
8ab1568352 Plan the ConfigKit consolidation sprint 2026-10-25 12:14:01 +00:00
8a99e61070 Defer clone traversal review to future work 2026-10-25 11:54:46 +00:00
de7513d488 Settle retaining the public loading APIs 2026-10-24 23:07:09 +00:00
26771dc03e Consolidate refactoring roadmaps and record open questions 2026-10-24 22:04:26 +00:00
fbaaaf18cd Draft the ConfigKit consolidation roadmap toward 1.0 2026-10-24 21:51:29 +00:00
c9574ef3c1 Record complexity audit findings 2026-10-24 19:54:13 +00:00
47 changed files with 4218 additions and 2610 deletions

View File

@@ -52,16 +52,6 @@ func fromEngineSourceInfo(value engine.SourceInfo) SourceInfo {
ImportSelections: fromEngineImportSelections(engine.SourceInfoImportSelectionsForFacade(value)),
}
}
func toEngineSourceInfo(value SourceInfo) engine.SourceInfo {
result := engine.SourceInfo{
ID: value.ID, Label: value.Label, Path: value.Path, Destination: toEnginePath(value.Destination),
Layer: engine.Layer(value.Layer), Kind: engine.SourceKind(value.Kind), UnderlayIndex: value.UnderlayIndex,
ImportedFrom: toEngineLocationPointer(value.ImportedFrom), VersionCheck: toEngineSourceVersionCheck(value.VersionCheck),
}
result = engine.SourceInfoWithImportSelections(result, toEngineImportSelections(value.ImportSelections))
return engine.SourceInfoWithPlacement(result, uint8(value.PlacementMode), value.DestinationUnresolved)
}
func fromEngineImportSelections(values []engine.ImportSelectionForFacade) []ImportSelection {
if values == nil {
return nil
@@ -76,15 +66,14 @@ func fromEngineImportSelections(values []engine.ImportSelectionForFacade) []Impo
return result
}
func toEngineImportSelections(values []ImportSelection) []engine.ImportSelectionForFacade {
func toEngineOverrides(values []Override) []engine.Override {
if values == nil {
return nil
}
result := make([]engine.ImportSelectionForFacade, len(values))
result := make([]engine.Override, len(values))
for index, value := range values {
result[index] = engine.ImportSelectionForFacade{
Path: toEnginePath(value.Path), Location: toEngineLocation(value.Location),
Selector: string(value.Selector), Outcome: string(value.Outcome),
result[index] = engine.Override{
Path: toEnginePath(value.Path), Value: value.Value, Flag: value.Flag,
}
}
return result
@@ -127,16 +116,6 @@ func fromEngineSourceVersionCheck(value *engine.SourceVersionCheck) *SourceVersi
Location: fromEngineLocationPointer(value.Location), Redacted: value.Redacted,
}
}
func toEngineSourceVersionCheck(value *SourceVersionCheck) *engine.SourceVersionCheck {
if value == nil {
return nil
}
return &engine.SourceVersionCheck{
Path: toEnginePath(value.Path),
Version: engine.Version{Present: value.Version.Present, Null: value.Version.Null, Text: value.Version.Text},
Location: toEngineLocationPointer(value.Location), Redacted: value.Redacted,
}
}
func fromEngineOrigin(value engine.Origin) Origin {
result := Origin{Layer: Layer(value.Layer), Locations: fromEngineLocations(value.Locations), Operation: ChangeKind(value.Operation)}
if value.Inheritance != nil {
@@ -150,19 +129,6 @@ func fromEngineOrigin(value engine.Origin) Origin {
}
return result
}
func toEngineOrigin(value Origin) engine.Origin {
result := engine.Origin{Layer: engine.Layer(value.Layer), Locations: toEngineLocations(value.Locations), Operation: engine.ChangeKind(value.Operation)}
if value.Inheritance != nil {
result.Inheritance = make([]engine.InheritanceStep, len(value.Inheritance))
for index, step := range value.Inheritance {
result.Inheritance[index] = engine.InheritanceStep{
From: toEnginePath(step.From), To: toEnginePath(step.To), Reference: toEnginePath(step.Reference),
Locations: toEngineLocations(step.Locations),
}
}
}
return result
}
func fromEngineDiagnostic(value engine.Diagnostic) Diagnostic {
return Diagnostic{Code: Code(value.Code), Severity: Severity(value.Severity), Path: fromEnginePath(value.Path), Message: value.Message, Primary: fromEngineLocationPointer(value.Primary), Related: fromEngineLocations(value.Related)}
}

View File

@@ -6,6 +6,30 @@ applications. Consumers still own their configuration structs, defaults, semanti
validation, schema-version policy, CLI parsing, runtime matching and conversion,
resource checks, remote acquisition, and service initialization.
## Choosing a loading entry point
- Use [`Load`](../../lifecycle.go) with one source for an ordinary configuration.
[`SearchConfig`](../../discovery.go) is also a source, so its explicit
discovery result can be passed to `Load` or used as the main input to
[`LoadLayered`](../../layered.go).
- Use `LoadLayered` when inputs must be composed in underlay groups before the
main source. Each [`Input`](../../layered.go) owns its source, filesystem,
destination, and import choice. The overlay filesystem context is selected
independently in [`LayeredOptions`](../../layered.go).
- Load-wide rules belong to [`Options`](../../lifecycle.go) or
[`LayeredOptions`](../../layered.go): the application definition (including
version policy), import defaults, and explicit overrides. These do not turn
on imports or discovery by themselves. See the
[configuration contract](../config.md#layered-loading) and
[discovery contract](../config.md#configuration-discovery) for exact rules.
The root [README example](../../README.md) shows the small `Load` path.
[`layered.go`](../../examples/consumers/layered.go) demonstrates the bundled
file and layered cases. For import-focused adoption, see the
[prompt catalog](../../examples/consumers/prompt_catalog.go); for explicit
discovery composed with sequences, see the
[sequence catalog](../../examples/consumers/sequence_catalog.go).
The maintained [`examples/consumers`](../../examples/consumers) package demonstrates
sanitized shapes using only ConfigKit and the Go standard library:

View File

@@ -189,6 +189,9 @@ remediation record are preserved in Git history at `0369111`, for example
[configuration discovery](config.md#configuration-discovery); the public
[discovery declarations](../discovery.go) own the Go API.
The [consolidation roadmap](roadmap/consolidation.md) owns the current refactoring
proposal and the associated complexity-audit findings.
Potential enhancements are tracked in [future features](roadmap/future.md).
Install the Go version required by `go.mod` or a compatible newer toolchain.
@@ -237,17 +240,17 @@ code changes. Use the component overview to find the corresponding implementatio
| Task | Behavioral test owners | Focused command |
| --- | --- | --- |
| YAML syntax, diagnostics, parser limits | [Adapter tests](../internal/yaml/parse_test.go), [source translation](../internal/engine/parse_test.go) | `go test ./internal/yaml` and `go test ./internal/engine -run '^TestParseSource'` |
| File/reader access, imports, globs, containment | [Sources](../internal/engine/source_test.go), [FIFO](../internal/engine/source_fifo_test.go), [imports](../internal/engine/imports_test.go), [relative destinations](../internal/engine/destination_import_test.go), [globs](../internal/engine/glob_test.go) | `go test ./internal/engine -run 'Source\|Import\|Glob\|Filesystem\|Nonregular\|RegularFile'` |
| File/reader access, imports, globs, containment | [Import internals](internal/imports.md), [source acquisition](../internal/engine/source_test.go), [FIFO](../internal/engine/source_fifo_test.go), [import traversal](../internal/engine/imports_test.go), [import evidence](../internal/engine/import_evidence_test.go), [relative destinations](../internal/engine/destination_import_test.go), [globs](../internal/engine/glob_test.go) | `go test ./internal/engine -run 'Source\|Import\|Glob\|Filesystem\|Nonregular\|RegularFile'` |
| Explicit main-file discovery and absent-input loading | [Candidate selection](../internal/engine/discovery_test.go), [load integration](../internal/engine/discovery_load_test.go), [public facade](../facade_discovery_test.go), and maintained [catalog](../examples/consumers/sequence_catalog_test.go) and [preset](../examples/consumers/discovery_test.go) examples | `go test ./internal/engine -run 'Discovery|Search'`, `go test . -run 'Search'`, and `go test ./examples/consumers -run 'WorkingDirectoryPreset|SequenceCatalog'` |
| Import defaults, empty-selection evidence, and dependency-ready ordering | [Defaults and facade evidence](../facade_import_defaults_test.go), [engine import evidence](../internal/engine/import_evidence_test.go), [scheduler](../internal/engine/dependency_order_test.go), [scheduling integrations](../internal/engine/imports_test.go) | `go test ./internal/engine -run 'Import|ScheduleDependencies|Inheritance'`, `go test . -run 'ImportDefaults|FacadeImport'`, and `go test ./examples/consumers` |
| Native filesystem and mapping sources | [Native acquisition](../internal/engine/native_filesystem_test.go), [filesystem facade](../facade_filesystem_test.go), [mapping facade](../facade_mapping_test.go) | `go test ./internal/engine -run 'NativeFilesystem'` and `go test . -run 'TestFacade'` |
| Composition, selected overlays, CLI precedence | [Composition](../internal/engine/compose_test.go), [overlay](../internal/engine/overlay_test.go), [selected overlays](../internal/engine/selected_overlay_test.go), [overrides](../internal/engine/override_test.go) | `go test ./internal/engine -run 'Compose\|Overlay\|Override'` |
| Sequence identity, declarations, and operations | [Identity](../internal/engine/sequence_identity_test.go), [registrations](../internal/engine/sequence_registration_test.go), [declarations](../internal/engine/sequence_declaration_test.go), [combination](../internal/engine/sequence_combine_test.go), [final facade validation](../facade_final_sequence_identity_test.go), and [consumer catalog](../examples/consumers/sequence_catalog_test.go) | `go test ./internal/engine -run 'Sequence\|Collection'`, `go test . -run 'SequenceIdentity\|SequenceRegistration'`, and `go test ./examples/consumers` |
| Sibling inheritance, provenance, history, sensitivity | [Expansion](../internal/engine/inheritance_test.go), [destination scopes](../internal/engine/inheritance_walk_test.go), [origins](../internal/engine/inheritance_origin_test.go), [history](../internal/engine/inheritance_history_test.go), [secrets](../internal/engine/inheritance_secret_test.go), [public loading](../facade_inheritance_test.go) | `go test ./internal/engine -run 'Inheritance\|Inherited'` and `go test . -run '^TestFacadeInheritance\|^TestFacadeInheritedSecrets'` |
| Sibling inheritance, provenance, history, sensitivity | [Implementation flow](internal/overview.md#implementation-flow), [expansion](../internal/engine/inheritance_test.go), [destination scopes](../internal/engine/inheritance_walk_test.go), [origins](../internal/engine/inheritance_origin_test.go), [history](../internal/engine/inheritance_history_test.go), [secrets](../internal/engine/inheritance_secret_test.go), [public loading](../facade_inheritance_test.go) | `go test ./internal/engine -run 'Inheritance\|Inherited'` and `go test . -run '^TestFacadeInheritance\|^TestFacadeInheritedSecrets'` |
| Layered loading and source equivalence | [Grouped lifecycle](../internal/engine/load_request_test.go), [layered facade](../facade_layered_test.go), [equivalence](../facade_equivalence_test.go) | `go test ./internal/engine -run '^TestGrouped'`, `go test . -run '^TestFacadeLayered'`, and `go test . -run '^TestEquivalentSourceForms'` |
| Input and relative-import destinations, mounted sources, and source ownership | [Destination engine tests](../internal/engine/destination_test.go), [tier tests](../internal/engine/destination_tiers_test.go), [import traversal and version tests](../internal/engine/destination_import_test.go), [accounting tests](../internal/engine/destination_accounting_test.go), [lifecycle inspection tests](../internal/engine/destination_import_lifecycle_test.go), [public source/error tests](../facade_destination_test.go), [resource contexts](../facade_resource_context_test.go), and [consumer examples](../examples/consumers/destinations_test.go) | `go test ./internal/engine -run 'Destination|Import|Glob|Mounted'`, `go test . -run 'Destination|Import|SourceContext|FacadeMounted'`, and `go test ./examples/consumers` |
| Shared source-version modes and evidence | [Engine source-version tests](../internal/engine/source_version_test.go), [public facade tests](../facade_source_version_mode_test.go), and [consumer example tests](../examples/consumers/layered_test.go) | `go test ./internal/engine -run 'SourceVersion'`, `go test . -run 'SourceVersion'`, and `go test ./examples/consumers` |
| Schema, defaults, projection, cloning | [Schema](../internal/engine/schema_test.go), [decoding](../internal/engine/decode_test.go), [projection](../internal/engine/projection_test.go) | `go test ./internal/engine -run 'Schema\|Decode\|Projection\|Clone\|Inline\|CompareDocument'` |
| Schema, defaults, projection, cloning | [Implementation flow](internal/overview.md#implementation-flow), [schema](../internal/engine/schema_test.go), [decoding](../internal/engine/decode_test.go), [projection and typed cloning](../internal/engine/projection_test.go), [deferred clone review](roadmap/future.md#4-clone-traversal-review) | `go test ./internal/engine -run 'Schema\|Decode\|Projection\|Clone\|Inline\|CompareDocument'` |
| Lifecycle, resolution, validation, callback ownership | [Loading](../internal/engine/load_test.go), [resolution](../internal/engine/resolution_test.go), [validation](../internal/engine/validation_test.go), [ownership](../internal/engine/callback_ownership_test.go), [independence](../internal/engine/independence_test.go) | `go test ./internal/engine -run 'Load\|Resolution\|Validation\|Validator\|Hook\|Callback\|Parallel'` |
| Paths, metadata, provenance | [Paths](../internal/engine/path_test.go), [metadata](../internal/engine/metadata_test.go), [provenance](../internal/engine/provenance_test.go) | `go test ./internal/engine -run 'Path\|Metadata\|Provenance\|Queries'` |
| Passive resource contexts | [Engine contexts](../internal/engine/resource_context_test.go), [public contexts](../facade_resource_context_test.go) | `go test ./internal/engine -run 'Resource'` and `go test . -run 'SourceContext'` |

View File

@@ -7,7 +7,10 @@ This note records implementation flow and private identities for maintainers.
## Discovery and acquisition
Each source's root directive is parsed into presence-aware declarations and
resolved against load-wide defaults. Candidate selection preserves a normalized
resolved against load-wide defaults. `acquisition.go` owns shared acquisition
policy, load accounting, and per-tier accounting; `import_acquisition.go` owns
reading candidate bytes, declaration traversal, and import admission. Candidate
selection in `import_selection.go` preserves a normalized
namespace path separately from canonical access identity: overlapping roots
coalesce the same candidate spelling in that namespace, while aliases reached
through distinct spellings retain separate inclusion routes for admission.
@@ -20,10 +23,14 @@ Directory traversal tracks canonical directories on the active traversal path.
It adds an identity when descending and removes it when the frame is popped, so
only a back-edge to an active directory is a traversal cycle; a later alias is
not mistaken for one. This is separate from import dependency-cycle detection.
For host sources, discovery uses rooted `Lstat` and canonical containment. Native
sources use `fs.ReadLinkFS` inspection when supplied; otherwise traversal follows
the capabilities and behavior of the provided `fs.FS`, without claiming a
general sandbox guarantee.
For host sources, discovery uses rooted `Lstat` and canonical containment.
`filesystem_context.go` owns resolved access facts and live access setup; the
acquisition operation closes its live handles before composition. Native sources
use `fs.ReadLinkFS` inspection when supplied; otherwise traversal follows the
capabilities and behavior of the provided `fs.FS`, without claiming a general
sandbox guarantee. `source_context.go` derives passive resource contexts from
resolved sources for later caller-owned reads; those records do not retain live
acquisition handles.
Acquisition remains authored: declarations are visited in source order and each
selected candidate set in namespace-path lexical order. This preserves source
@@ -78,10 +85,14 @@ mapping order in its output.
These scopes do not flatten into one global graph, and independent precedence
tiers remain separate.
The main owners are [`import_selection.go`](../../internal/engine/import_selection.go),
The owners are [`acquisition.go`](../../internal/engine/acquisition.go),
[`import_selection.go`](../../internal/engine/import_selection.go),
[`import_acquisition.go`](../../internal/engine/import_acquisition.go),
[`directory_walk.go`](../../internal/engine/directory_walk.go),
[`import_graph.go`](../../internal/engine/import_graph.go),
[`import_projection.go`](../../internal/engine/import_projection.go),
[`dependency_order.go`](../../internal/engine/dependency_order.go), and
[`inheritance_schedule.go`](../../internal/engine/inheritance_schedule.go).
Their focused behavior is covered by import-selection, scheduler, and scheduling
integration tests under `internal/engine`.
Their focused behavior is covered by import-selection, acquisition, scheduler,
and scheduling integration tests under `internal/engine`; see the
[development test map](../development.md#task-to-test-map).

View File

@@ -87,3 +87,58 @@ operations and CLI ordering. [ADR 0019](../adr/0019-explicit-configuration-disco
records explicit main-file discovery. The [import internals](imports.md)
document candidate and namespace identities, discovery and scheduling, and the
bounded algorithms.
## Implementation flow
The root facade converts caller-owned values into engine-owned requests. The
ordinary and grouped entry points share a normalized `loadRequest`: see
[`lifecycle.go`](../../lifecycle.go), [`layered.go`](../../layered.go), and
[`load_request.go`](../../internal/engine/load_request.go). Load-wide definition,
version, default-import, and override policy stays distinct from each input's
source, filesystem, destination, and import settings.
[`acquisition.go`](../../internal/engine/acquisition.go) defines acquisition
policy and load-wide and per-tier accounting state. Each input is acquired under
that policy; grouped requests retain acquired underlay scopes until their tier is
composed in [`load_request.go`](../../internal/engine/load_request.go). Live
filesystem access handles are confined to acquisition and closed there. Resolved
filesystem facts used to authorize access are separate from passively retained
resource contexts: see
[`filesystem_context.go`](../../internal/engine/filesystem_context.go) and
[`source_context.go`](../../internal/engine/source_context.go).
Import responsibilities are split across candidate selection and path
normalization in [`import_selection.go`](../../internal/engine/import_selection.go),
source reading and traversal admission in
[`import_acquisition.go`](../../internal/engine/import_acquisition.go),
directory walking in [`directory_walk.go`](../../internal/engine/directory_walk.go),
dependency graph scheduling in [`import_graph.go`](../../internal/engine/import_graph.go),
and projection of acquired evidence into final source coordinates in
[`import_projection.go`](../../internal/engine/import_projection.go). The
[import internals](imports.md) page describes the boundaries and identities.
After acquisition handles are closed, grouped composition and inheritance run on
the retained ordered documents. `inheritance.go` coordinates traversal and
[`inheritance_schedule.go`](../../internal/engine/inheritance_schedule.go) applies
dependency-ready templates;
[`inheritance_prepare.go`](../../internal/engine/inheritance_prepare.go)
prepares child and parent inputs,
[`inheritance_combine.go`](../../internal/engine/inheritance_combine.go)
returns the combined value with its origin, policy, sequence, and diagnostic
evidence, and [`inheritance_finalize.go`](../../internal/engine/inheritance_finalize.go)
accounts for and finalizes the result. Projection and schema work have separate
operation owners: [`value_operation.go`](../../internal/engine/value_operation.go)
holds per-operation state, [`projection.go`](../../internal/engine/projection.go)
builds projected values, [`schema.go`](../../internal/engine/schema.go) owns
descriptor caching and type policy, and
[`schema_fields.go`](../../internal/engine/schema_fields.go) owns ordinary and
inline field assembly. [`decode.go`](../../internal/engine/decode.go) consumes
those descriptors for typed decoding.
Behavioral ownership follows the same handoffs: imports and grouped lifetimes are
covered by `imports_test.go`, `load_request_test.go`, and focused source,
discovery, and destination tests; inheritance by the inheritance test files;
reflection behavior by `schema_test.go`, `projection_test.go`, and
`decode_test.go`. The [development test map](../development.md#task-to-test-map)
links focused commands. Typed clone extraction remains deferred in
[`future.md`](../roadmap/future.md#4-clone-traversal-review).

View File

@@ -1,222 +0,0 @@
# Internal Complexity Refactoring Roadmap
Status: proposed target state. This roadmap defines a behavior-preserving
refactor; a later implementation plan will define stages and exact private
signatures and file names.
## Purpose
Make ConfigKit's import, inheritance, and projection implementation easier to
understand, review, and change while preserving its public APIs and configuration
behavior. Concentrate on functions that coordinate several responsibilities and
carry substantial shared state.
The target is clearer responsibility and ownership. Adding a few well-named
helpers, private records, or focused files can serve that target even when total
line count increases. Complexity measurements help locate candidates; they do
not establish function-size limits, score thresholds, or release gates.
The [architecture policy](../policy/architecture.md) owns package and ownership
constraints. The [configuration contract](../config.md) and exported Go comments
remain authoritative for consumer behavior.
## Current Baseline
Reviewed baseline: `7a83704f43ef9870db33661231cdff5d6ee502f7`.
The root facade delegates to a cohesive private engine, supported by the ordered
document representation and YAML adapter. Most functions are small. The
complexity audit identified these concentrated maintenance costs:
| Area | Current implementation | Required improvement |
| --- | --- | --- |
| Imports | `imports.go` spans acquisition, item-local composition, and evidence projection. Private loading variants pass long positional argument lists. Source admission and directory selection contain substantial mixed control flow. | Give import entry points explicit, cohesive inputs and separate the responsibilities of acquisition, scope assembly, and evidence projection. |
| Inheritance | `applyInheritanceScopeTemplates` coordinates payload preparation, parent combination, policy ownership, origins, history, and accounting in one executor. | Keep one readable executor with focused operations and explicit state handoffs. |
| Projection | `valueOperation.project` combines type dispatch with struct flattening, map and sequence traversal, scalar conversion, sensitivity, and accounting. | Retain clear dispatch while moving substantial value-kind handling into focused helpers. |
The implemented inventory belongs to the
[component overview](../internal/overview.md). Existing import flow and scheduling
are described in [import internals](../internal/imports.md).
## Target Architecture
The refactor stays within `internal/engine`. The root facade, `internal/document`,
and `internal/yaml` retain their existing responsibilities and dependency
direction. Focused files and ordinary private functions express the new internal
organization. Package extraction is outside this roadmap's selected design.
`executeLoadRequest` remains the visible coordinator of the shared load lifecycle.
Its phase order should remain easy to follow. Helpers expose meaningful work and
results rather than distributing that order across a generic execution framework.
Existing shared mechanisms remain shared: recursive mapping combination,
identified-sequence operations, dependency-ready scheduling, schema descriptors,
and projection/accounting state. New helper boundaries make their ownership and
use clearer without creating parallel implementations.
### Imports: Cohesive Inputs And Scope Ownership
Import loading has a clear private entry surface using named request/context
fields for related inputs. Reuse the existing load-request, acquisition-scope,
and traversal models where they fit. Consolidate forwarding variants whose only
purpose is passing another optional positional argument through the chain.
Retain convenience helpers when they express a useful, distinct operation.
Keep these state lifetimes identifiable:
- Load-wide state: accounting, shared inclusion/repetition information,
namespaces, and source-version processing.
- Acquisition requests: source and filesystem context, placement, declarations,
applicable policy, and the relevant assembly handoff.
- Composition scopes: peer contributions, item-local helpers, record ownership,
and pending declarations.
- Evidence: physical source records, effective destinations, origins, and retained
history projections.
Group acquisition, pending-item assembly, and evidence projection in focused
files. A caller should be able to identify which operation acquires resources,
which composes already acquired content, and which projects retained evidence.
Shared mutable state has a visible owner; a request record need not duplicate
state already held by that owner.
Within dense admission, directory-selection, and declaration-parsing routines,
extract coherent operations where that reduces nested, mixed responsibilities.
Preserve the existing selection algorithms and host/native filesystem behavior.
Resource ownership and closure remain explicit across success and failure paths.
The resulting flow preserves the distinction among acquisition order, strict
scope scheduling, and precedence. Completing a helper scope before placing its
item remains part of the same import flow. Ordinary and layered loading continue
to share the applicable mechanisms.
### Inheritance: One Executor With Explicit Phases
Keep the mapping and identified-sequence adapters and their shared template
executor. The executor coordinates focused operations for:
- Preparing child payload, directives, and destination-local state.
- Preparing the resolved parent payload and applicable inherited policy.
- Combining values through the existing mapping and sequence mechanisms.
- Producing policy, origin, replacement, and replay evidence from that combination.
- Accounting for expansion and retained history under the existing budgets.
These are responsibility boundaries, not a required helper count or a new
scheduler. Small private result records are appropriate when related outputs
otherwise travel through long argument lists or implicit mutations.
Authored templates, resolved values, scoped policy, and inspection evidence remain
distinguishable. Immutable borrowed inputs and owned mutable results have clear
lifetimes. Nested instructions remain available for destination-local evaluation.
For identified-record inheritance, the child's own identity fields and origins
remain intact; the parent's identity fields are excluded from inherited payload
and its history, as the current contract requires.
One sequence-operation result continues to drive value construction, record
ownership, origins, and historical position projection. Helpers consume that
evidence directly. Scheduling remains dependency-ready, enclosing relationships
resolve before nested ones, and assembled sequence order remains unchanged.
### Projection: Explicit Value-Kind Handlers
Keep `projectConfiguration` as the schema-checked entry and `valueOperation` as
the owner of per-operation traversal and accounting state. The central dispatch
clearly selects the appropriate handling for pointers, dynamic values, structs,
maps, sequences, and scalars.
Substantial struct, map, and sequence work lives in focused handlers. Struct
projection retains declaration order, inline flattening, collision diagnostics,
and sensitivity propagation. Map projection retains deterministic key ordering.
Scalar conversion retains the existing number, duration, text-codec, and secret
reference representations.
Depth and node accounting retain their existing meaning, including flattened
containers. Cycle tracking stays active through recursive child traversal and
is balanced on error paths as well as success. Passing state through a helper
does not introduce another full-tree projection or an unnecessary ownership copy.
Cloning and source-value construction remain distinct operations with their
existing semantics. Reuse small helpers when the semantics coincide; keep the
projection refactor centered on making the existing traversal readable.
## Behavior And Scope Preservation
The implementation preserves:
- Root-owned public types, signatures, supported import paths, and facade boundaries.
- Supported source forms, opt-ins, import controls, placement, acquisition and
composition ordering, and source/version evidence.
- Strict peer composition, precedence operations, identity agreement and lifetime,
record ownership, and destination-local inheritance.
- The load lifecycle, including typed defaults, post-decode CLI assignment,
recorded application resolution, and final validation.
- Explicit presence, provenance, history, sensitivity, resource contexts, and
detached publication/query results.
- Established diagnostic codes, semantic paths, source coordinates, ordering,
resource lifetime, and existing limit behavior.
The owning contracts define the extent of these obligations. Incidental private
call sequences, exact helper names, and allocation counts do not acquire new
guarantees through this roadmap. The work adds no consumer options, configuration
syntax, dependencies, or complexity-based gates.
## Relationship To Remediation And Other Work
The completed remediation record at `ea46001:docs/roadmap/remediation.md`
documents the identified correctness and algorithmic efficiency findings.
Several overlap these files, particularly item-scope application matching and
inheritance policy localization.
Use the documented contract and completed remediation as the behavioral baseline.
Keep corrective changes reviewable under their findings, and retain their
regressions through this refactor. The later implementation plan should coordinate
overlapping work and select its starting revision rather than duplicating the
remediation plan here. These structural changes do not close a finding merely
because its function moved or became smaller.
[Configuration discovery](../config.md#configuration-discovery) is implemented.
Import cleanup should preserve its source-acquisition boundary and behavior as
part of this refactor.
## Work Needed And Validation
Implementation needs to consolidate import inputs and adapters, separate import
responsibilities, extract inheritance operations with explicit outputs, and
decompose projection handling. Remove superseded private plumbing as its callers
migrate. Review the resulting call flow for repeated traversals, policy scans,
copies, or hidden state transitions introduced by extraction.
Reuse the existing behavioral suites under the
[development guide](../development.md) and [testing policy](../policy/testing.md):
- Imports: selection, scheduling, source/version evidence, placement, helper scopes,
ordinary/layered equivalence, filesystem failures, and resource lifetime.
- Inheritance: mapping and record relationships, nested destination resolution,
cancellation and replacement, policy ownership, history, sensitivity, and
independent loads.
- Projection: schema/decoding agreement, inline fields, text codecs, nil/empty
values, cycles and budgets, secrets, and detached results.
Add focused regressions only where a consequential handoff risk lacks coverage.
Tests should continue to describe behavior as private functions and files move.
Use the existing everyday check and applicable ownership race checks during
implementation. Complexity measurements can inform review; acceptance does not
depend on numerical scores or a new benchmark campaign.
Update the component overview and import internals to describe the implemented
ownership and file responsibilities when the refactor lands. Update contributor
navigation and test-owner references where moves affect them. Public behavior
continues to be defined by its current canonical owners.
## Completion Criteria
- Import callers pass cohesive named inputs, with clearly owned load-wide and
scope-local state and fewer redundant forwarding variants.
- Import acquisition, item assembly, and evidence projection are easy to locate
and follow across focused private files.
- Inheritance orchestration exposes value, policy, evidence, and accounting work
without requiring readers to reconstruct their interaction from one large body.
- Projection dispatch and substantial value-kind handlers have clear boundaries
and share the original operation state correctly.
- Existing APIs and contracted behavior, including completed remediation, remain
covered by the relevant behavioral and ownership checks.
- Internal documentation matches the resulting implementation.

View File

@@ -0,0 +1,481 @@
# ConfigKit Consolidation Toward 1.0
Status: selected target state; implementation pending. This document defines the
refactor's scope and rationale; it does not authorize a release. The scope
questions are settled. The [implementation plan](implementation.md) owns the
concrete steps, sequencing, and implementation progress.
## Purpose
Consolidate the machinery introduced by recent import, collection, inheritance,
and discovery work so ConfigKit is easier to understand, maintain, and adopt
before its public contracts become stable. Success means fewer independently
maintained paths through the same behavior, clearer state ownership, and concise
consumer workflows. Reducing a function's line count is useful only when it
serves those outcomes.
The review considered public API consolidation as well as internal extraction.
The settled public direction retains the existing simple and layered loading APIs,
consolidates their private translation and acquisition machinery, and removes
unused reverse inspection conversions. No public signature or configuration
semantic change is planned in this effort. This is an assessment of the reviewed
consumer workflows, not a requirement to preserve every beta API in perpetuity.
## Evidence And Baseline
Reviewed worktree: `c9574ef`, immediately after the `v0.10.0` release at `5959537`.
This document replaces the earlier `complexity.md` roadmap and `remediation.md`
audit record. Their recommendations and relevant evidence are incorporated here;
the original documents remain available in Git history:
```sh
git show fbaaaf1:docs/roadmap/complexity.md
git show fbaaaf1:docs/roadmap/remediation.md
```
The earlier roadmap reviewed `7a83704f43ef9870db33661231cdff5d6ee502f7`, before
discovery. The audit reviewed `5959537d1adc2516e7f7e8881df72b707ec48c3f`, including
implemented discovery. The consolidation review checked their recommendations
against that newer baseline and also considered the public API.
This review inspected the current public load/input/discovery declarations and
maintained consumer examples, the shared request executor and acquisition flow,
import parsing and selection, inheritance execution, reflective projection and
cloning, schema field construction, and filesystem-context derivation. Graph
caller traces and text-reference checks confirmed the unused reverse conversion
chain. This was source review, not a new runtime correctness or performance audit.
Neither this review nor the audit established a runtime correctness defect.
### Audit Measurements
The audit refreshed the codebase-memory graph and used function size, cognitive
complexity, branch counts, and parameter counts to rank candidates for source
inspection. It indexed 901 non-test functions/methods and 928 test-file
declarations. Its fast index excludes examples, documentation, and scripts; a
tracked-file inventory covered those separately. Standalone `gocyclo`, `gocognit`,
`scc`, and `cloc` tools were unavailable.
These are the graph tool's reported measurements at the audited revision.
Function spans include comments and blank lines. The scores support relative
triage; they are not standardized acceptance thresholds or evidence of a defect.
| Function | Source | Lines | Cognitive score |
| --- | --- | ---: | ---: |
| `valueOperation.project` | [projection.go](../../internal/engine/projection.go) | 195 | 174 |
| `parseImportDeclarationSpec` | [import_spec.go](../../internal/engine/import_spec.go) | 127 | 129 |
| `valueOperation.clone` | [clone.go](../../internal/engine/clone.go) | 130 | 104 |
| `schemaBuilder.describeFields` | [schema.go](../../internal/engine/schema.go) | 99 | 100 |
| `assembleBaseInputTrackedForRole` | [load_input.go](../../internal/engine/load_input.go) | 188 | 96 |
| `selectDirectoryImportCandidates` | [import_selection.go](../../internal/engine/import_selection.go) | 154 | 96 |
| `valueBuilder.build` | [value.go](../../internal/engine/value.go) | 104 | 93 |
| `decodeValue` | [decode.go](../../internal/engine/decode.go) | 115 | 85 |
| `applyInheritanceScopeTemplates` | [inheritance_schedule.go](../../internal/engine/inheritance_schedule.go) | 219 | 75 |
| `contextsForAcquiredSources` | [source_context.go](../../internal/engine/source_context.go) | 102 | 72 |
At that revision, the engine contained 88 production Go files and approximately
20,144 physical lines; the root facade contained 16 files and 1,861 lines.
`imports.go` was the largest production file at 1,195 lines. These measurements
locate concentrated responsibilities. They do not establish a need for more
packages or make the smaller document/YAML visitors and explicit facade
conversions maintenance defects.
### Findings And Recommended Treatment
The finding numbers below preserve the audit's identifiers. The selected work
remains unimplemented; the cloning portion of finding 3 is deferred to future
work. Retiring the original documents does not mark their findings implemented.
| Audit finding | Maintenance priority | Assessment and recommended treatment |
| --- | --- | --- |
| 1. Import entry points and responsibilities | Higher | Main refactor. Include shared acquisition normalization, import-default ownership, declaration parsing, and directory admission. |
| 2. Shared inheritance executor | Higher | Main refactor. Expose meaningful phases and explicit combination results while retaining one executor. |
| 3. Projection and cloning | Higher for projection; cloning deferred | Main refactor for projection. Clone traversal review is tracked separately in [future.md](future.md#4-clone-traversal-review). |
| 4. Schema field discovery and inline assembly | Medium | Include a focused refactor coordinated with projection. Keep one schema builder and its two descriptor views. |
| 5. Resolved filesystem context | Medium | Include alongside acquisition work. Reuse resolved facts without merging resource lifetimes or changing validation timing. |
| 6. Unused reverse inspection conversions | Small, independently reviewable cleanup | Include in the root facade and engine. Retain useful forward-conversion and detached-result coverage. |
Findings 1–3 confirm the earlier roadmap's main opportunities; findings 4–6 add
focused work. The following sections own their rationale and intended changes.
Implementation and validation will establish completion of the selected work;
the deferred clone review retains its separate disposition.
Completed correctness and algorithmic remediations remain part of the baseline,
including sequence records preserved at `ea46001:docs/roadmap/remediation.md`
and discovery records preserved at `0369111:docs/roadmap/remediation.md`.
Item-scope application matching and inheritance policy localization are examples
of completed corrections that overlap this work and whose regressions should
remain protected.
The [architecture policy](../policy/architecture.md),
[configuration contract](../config.md), and exported declaration comments own
existing constraints and behavior. The
[component overview](../internal/overview.md) and
[import internals](../internal/imports.md) locate implementation responsibilities.
## Consumer-Facing Target
### Keep Simple Loading Simple And Advanced Inputs Explicit
The current APIs already provide progressive setup:
- A simple consumer calls `Load[Config](File(path))`.
- A consumer with defaults, imports, or resolution supplies `Options[Config]`.
- Explicit discovery produces an ordinary `Source`, so it uses the same loading
call and lifecycle.
- A layered consumer supplies an `Input` and `LayeredOptions[Config]`, with
`Underlay.Inputs` distinguishing strict peers from precedence tiers.
The two loading functions already converge on `loadRequest` and
`executeLoadRequest`. Their public differences have meaning: ordinary options
include the one input's controls; layered loading has per-input controls and a
separate overlay filesystem. Ordinary overlay context follows its selected
source, whereas layered overlay context is independently specified. These are
observable conventions, not interchangeable forwarding spellings.
**Decision (OQ-1, settled):** retain `Load`, `LoadLayered`, `Options`, `LayeredOptions`,
`Input`, and `Underlay`. Remove repeated translation and option reconstruction
behind them. Keep common application rules in `Definition`; do not require
applications to build a runtime loader or manually coordinate phases.
A single public request type could place main input, underlays, and common
options together. It would reduce duplicated option fields and make promotion
from ordinary to layered loading more uniform. A convenience entry could preserve
the simplest call. Such a design would still need to resolve the existing
overlay-context differences and demonstrate a clearer consumer model. The
maintained examples do not show enough consumer benefit to justify that redesign
in this effort. Retaining the current APIs accepts some duplicated public option
fields and setup changes when adopting underlays; internal consolidation and
clearer guidance address the observed maintenance needs. The decision is based on
the resulting API, rather than the amount of refactoring needed to achieve it.
### Preserve Useful Convenience And Ownership Boundaries
| Surface reviewed | Recommended public end state |
| --- | --- |
| `Source`, `Filesystem`, `Search`, filename groups and presets | Retain typed, reusable source/search descriptions. Discovery selects a concrete source during loading; applications do not loop over complete loads. |
| `FilesystemContext` | Retain the public description and current host/native rules. Consolidate its resolved private representation; do not expose acquisition handles or engine context records. |
| `SequenceIdentityAt` and `SequenceIdentityMatching` | Retain the exact-path convenience and dynamic selector form over the common registration model. Both serve distinct consumer shapes. |
| `Definition`, resolution and validation callbacks | Retain application-owned domain rules inside one recorded lifecycle. Preserve the distinction between mutable recorded resolution and read-only validation. |
| Results, metadata, explanations, source context and diff | Retain detached, safe inspection with consumer-owned presentation. Remove unused inverse conversions without removing supported queries. |
| Root wrappers and private conversions | Retain root-owned public types and explicit ownership conversion under ADR 0006. Consolidate repeated real conversions and delete unused ones. |
The consumer-facing work is to make the ordinary path easier to find and show
how additional features compose, using the existing maintained examples. Update
the adoption guide with a concise choice between simple loading, discovery, and
layered loading, and show which options belong to an input versus a whole load.
Keep exact behavior in its canonical owners. Examples should make application
rules recognizable and avoid requiring consumers to recreate generic merging,
presence, or provenance machinery.
Use the current simple README call, bundled-file and layered catalog examples,
the import-focused prompt catalog, and the discovered sequence catalog to assess
the end state. The latter exercises imports, underlays, independent overlay
context, CLI assignment, and recorded resolution in one workflow. A high number
of options in that example does not by itself justify a new abstraction.
## Internal Target
Keep the root facade, cohesive `internal/engine`, ordered document model, and
private YAML adapter. Use ordinary private records, functions, and focused files
to express responsibilities. Existing mapping combination, sequence combination,
dependency scheduling, schema descriptors, and traversal accounting remain shared.
`executeLoadRequest` stays the visible lifecycle coordinator. It should show
acquisition, assembly, inheritance, decoding/CLI assignment, resolution, secret
processing, validation, and publication in their established order. Extract a
phase only when it gives that work a clear input, output, or ownership boundary.
Do not replace the visible lifecycle with a registry of configurable stages.
### 1. Normalize Loading Inputs And Consolidate Import Acquisition
The current executor constructs an engine `Options[T]` from `loadRequest`, and
underlay acquisition constructs `Options[struct{}]` to call the same input
assembler. `assembleBaseInputTrackedForRole` then combines discovery, directive
processing, version rules, imports, placement, accounting, and contexts. Import
entry points carry long positional lists through several forwarding variants.
This repeats the representation of information already normalized for a load.
In [imports.go](../../internal/engine/imports.go), the
`loadImportedDocumentsTrackedWithVersion`,
`loadImportedDocumentsTrackedWithProcessing`, and
`loadImportedDocumentsTrackedWithProcessingDeferred` variants have corresponding
`loadImportedWithAccess...` variants. Both final signatures carry 13 parameters.
Placement, processing results, sequence policy, accounting, repetition tracking,
namespace identity, and deferred assembly must stay aligned through the chain.
The same file's `importTraversal.visitPath` combines source-admission decisions
in 174 lines, alongside acquisition, pending-item assembly, and evidence work.
Introduce cohesive private acquisition inputs and explicit state ownership,
reusing `loadRequest`, `loadInput`, `acquisitionScope`, and `importTraversal`
where they fit. Translate ordinary and layered public declarations into the
shared loading model once. Acquisition should receive the source, input role,
placement, context, and policies it needs without a fabricated application type
or a reconstructed broad options object. Small adapters may preserve meaningful
entry operations while callers migrate; remove obsolete forwarding variants
within the completed refactor.
Separate immutable policy, mutable processing state, and results. Move import
defaults out of `sourceVersionState` and into the load/acquisition policy owner.
Version state continues to own version processing, projected effective version
location, evidence, and diagnostics. Local and effective version rules have
different meanings and must remain distinguishable.
The state boundaries are especially important:
| Lifetime or scope | State and responsibility |
| --- | --- |
| Whole load | Aggregate resource accounting, application definition/registrations, import defaults, shared version evidence and diagnostics. |
| One strict tier | Repetition tracking shared by that tier's inputs, namespace scheduling anchors, and its inclusion/composition graph relationship. A later tier may reuse a file. |
| Active import traversal | Active routes and inclusion edges, filesystem access ownership, and cycle detection, including edges crossing item-scope boundaries. |
| One composition scope | Peer contributions, pending collection declarations, item-local helpers, and deferred item placement. Nested scopes do not become separate precedence tiers. |
| Retained result | Source records, effective destinations, origins, history, and passive resource-context descriptions. |
Keep repetition tracking tier-scoped. In the current code,
`acquireUnderlayGroupScopes` creates a fresh visited map per group. Preserve valid
reuse across tiers while keeping shared aggregate limits. Namespace equality,
candidate spelling, and stable scheduling keys also serve different purposes.
Give acquisition, already-acquired scope assembly, and evidence projection
focused homes. Keep discovery at the acquisition boundary: resolve a search once
per input consumption, then process the concrete file or absence. Preserve
source IDs and source-local version processing in acquisition order, even when
composition uses a different dependency-ready order. The layered main is acquired
before underlays; lower-tier identity can still inform its later strict assembly.
Within `parseImportDeclarationSpec`, extract boolean/collection/placement field
handling and selector-value parsing where it clarifies responsibilities. Keep
field presence, authored error locations, selector applicability, and the order
of consequential validation visible. Within `selectDirectoryImportCandidates` and
`importTraversal.visitPath`, separate admission and descent decisions from
traversal orchestration where they represent distinct work. Retain the existing
root deduplication, candidate union, exclusions, sorting, and active-path cycle
rules. No shared abstraction should conflate literal first-match main discovery
with multi-file import selection.
### 2. Share Resolved Filesystem Facts Without Sharing Live Handles
`importFilesystem`, `nativeFilesystemAccess`, and `contextsForAcquiredSources`
derive overlapping root/base/namespace facts. The latter reconstructs context
from combinations of source kind and optional fields, including absent input.
Discovery and independent layered overlays make these combinations consequential.
Context construction is used by ordinary acquisition and both selected-overlay
paths, so those callers need to use the same applicable derivation rules.
Use a small engine-owned resolved-context description for the facts actually
needed by acquisition and passive views: namespace identity/kind, resolved root,
source base, and selected source path when present. Preserve the distinction
between host paths and native slash paths. Reuse already resolved facts where
available instead of repeatedly deriving them from the original options.
Resolution and validation can occur at different existing lifecycle points.
The record must not force every source to perform all filesystem checks early.
Context descriptions, library-owned acquisition handles, and returned passive
resource views have different lifetimes. Closing acquisition must remain
explicit on success and failure; retaining context must not retain an open root
handle or assume ownership of a borrowed `fs.FS`.
Cover selected and absent discovery outcomes, root-only absent host context,
native sources, in-memory inputs with explicit context, and independently rooted
overlays. Passive context remains associated with the source that contributed
the value, including retained underlay and inherited values. Use the acquired
source's canonical identity where available. Preserve containment failures,
source associations, deferred search bases, and terminal selected-file errors.
Acquisition and context handling use the same private handoff rather than
maintaining parallel models for overlapping callers. The description does not
replace the public `FilesystemContext` or `ResourceContext` contracts.
### 3. Make Inheritance Combination Results Explicit
Keep the mapping and identified-record adapters and one shared executor.
`applyInheritanceScopeTemplates` currently prepares templates, mutates values
through merge callbacks, projects policy and origins, records replacements and
replay evidence, and enforces budgets in one body. The callbacks mutate several
surrounding result collections, making a combination difficult to review alone.
Give the executor named operations for child/control preparation, parent payload
and policy preparation, combination, evidence construction, and retained-node
accounting. Use a small private result for outputs of one combination where it
replaces unrelated surrounding mutations. Value, policy, origin, replacement,
and sequence-position evidence should come from the same combination outcome;
do not rerun a merge to reconstruct its history.
Keep borrowed authored templates separate from owned resolved payloads and from
retained history snapshots. Parent identity exclusion and child identity/origin
preservation must apply consistently to values and history. Preserve nested
unresolved instructions for destination-local evaluation, policy retirement on
actual replacement, cancellation, and source-version movement through sequences.
The shared Kahn scheduler with stable keys remains unchanged. Resolve enclosing
relationships before nested relationships and retain assembled record order.
Preserve expansion limits and retained-history accounting for shared nodes;
extraction must not charge shared evidence twice or make an unbounded snapshot.
### 4. Separate Projection Handlers And Clarify Schema Assembly
Keep `projectConfiguration` as the schema-checked entry and `valueOperation` as
the operation-state owner. Move substantial struct, map, and sequence projection
into focused handlers. Keep pointer/dynamic dispatch and scalar representation
choices visible. All handlers share the same cycle, depth, and node state.
Struct projection needs particular care: physical members drive traversal,
flattened fields drive name lookup and sensitivity, and an inline container is
not retained as an extra output node. Preserve declaration order, deterministic
map order, nil/empty distinctions, collision diagnostics, codec failures, and
secret classification. Scalar projection retains the existing number, duration,
text-codec, and secret-reference representations. Keep cycle entries active
through child traversal and balance their release on every exit.
In `schemaBuilder.describeFields`, separate field discovery and tag parsing from
inline struct assembly, inline map assembly, and ordinary field registration.
Retain one builder/cache and its recursion detection. Make the paired updates to
`members`, flattened `fields`, `fieldByName`, and `inlineMap` easy to follow.
Keep index-path construction and inherited secret flags intact. These descriptor
views are both needed; combining them into one collection would shift complexity
into decoding and projection.
Keep pointer-chain and recursive-inline decisions explicit, preserve supported
field/tag forms and ambiguity paths, and retain cache/building-state behavior.
Clone traversal review and cleanup are deferred to
[future.md](future.md#4-clone-traversal-review). This effort preserves cloning's
existing behavior while changing the shared schema and projection machinery.
Keep projection's document-building semantics distinct from typed copying.
The mostly explicit dispatch in `decodeValue` and `valueBuilder.build` does not
warrant a generic reflection visitor solely because it has a high complexity
score.
### 5. Remove Unused Inspection Conversions And Consolidate Real Adapters
Remove `toEngineSourceInfo`, `toEngineOrigin`, `toEngineSourceVersionCheck`, and
`toEngineImportSelections` from [conversion.go](../../conversion.go), along with
the engine adapters used only by that chain.
At the reviewed revision, the three callers of `toEngineSourceInfo` are tests;
`toEngineOrigin` has none. `SourceInfoWithPlacement`,
`SourceInfoWithImportSelections`, `importSelectorFromString`, and
`importSelectionOutcomeFromString` in [records.go](../../internal/engine/records.go)
have no production role beyond that reverse chain. Consumers receive inspection
records; production facade operations do not send those records back to the
engine. The unused inverse path adds another ownership mapping to maintain as
inspection evidence grows. Recheck callers at the implementation baseline before
removal.
Rewrite the affected [record tests](../../records_test.go) around real
engine-to-public conversion and detached public inspection. Preserve useful
placement, version evidence, import-selection, and copy-isolation assertions.
Avoid manufacturing replacement production setters solely to keep an old
round-trip fixture working. Existing facade loads already provide genuine sources
and inspection records.
Retain reverse conversions that serve real inputs: `toEngineDiagnostics`, for
example, supports `WriteDiagnostics`. Consolidate repeated override/input
conversion in `toEngineOptions` and `toEngineLayeredOptions` where semantics
match. Keep explicit root-to-engine ownership conversion and the public-boundary
checks. A reflective transport framework or public engine aliases would obscure
the boundary rather than simplify it.
## Scope, Compatibility, And Policy Consequences
The proposed work spans private engine code, private root-facade conversions,
affected tests, consumer guidance, and implementation documentation. Under the
settled answer to [OQ-1](#oq-1-should-the-public-loading-apis-change-in-this-effort),
preserve public types/signatures and the behavior owned by the current contracts,
including:
- Acquisition versus composition ordering, strict scopes versus precedence
tiers, identity ownership, placement, and source-version protection.
- Discovery exhaustion and error behavior, source lifetime, passive resource
contexts, and host/native namespace distinctions.
- Defaults, post-decode CLI assignment, recorded resolution, secret handling,
final registered-identity enforcement, and semantic validation.
- Explicit presence, provenance/history, sensitivity, safe diagnostics and diff,
frozen inspection, and detached publication/query results.
- Contracted diagnostic codes, paths, coordinates, ordering, and resource limits.
Do not turn incidental helper order, exact private representations, numerical
complexity scores, or allocation counts into new guarantees. Keep the existing
algorithmic remediations effective; watch for repeated whole-tree traversals,
policy scans, or copies introduced by helper extraction.
The recommendation fits the existing architecture policy and ADRs 0006,
0015, 0017, 0018, and 0019. It needs no new architectural ownership decision or
policy revision merely to introduce private helpers. Update the component
overview and import internals when code moves. Accepted ADR decision content
remains unchanged. A public loading-model redesign is outside this effort's
settled scope.
Pre-1.0 public changes are possible under the [release policy](../policy/release.md),
but they must have a consumer benefit and documented impact. This draft does not
propose a parallel compatibility API or temporary migration scaffold. It also
does not make refactoring completion a new release gate or establish that 1.0
is otherwise ready. Future performance experiments remain in
[future.md](future.md#3-performance-experiments); no benchmark campaign was run
or is required to claim readability improvements.
## Validation And Completion
Use the [development guide](../development.md) and
[testing policy](../policy/testing.md) for focused checks and the ordinary full
development check. Preserve existing behavior tests; rewrite tests coupled only
to removed plumbing. Add a test when a consequential handoff risk lacks coverage,
not for every new helper.
| Work area | Existing protection and review focus |
| --- | --- |
| Acquisition and imports | Selection, defaults, scheduling, relative/item placement, source-version evidence, aggregate limits, repetition within versus across tiers, and ordinary/layered equivalence. |
| Resolved contexts | Host/native access, discovery selected/absent outcomes, terminal context failures, source-relative resources, independent overlay context, and handle closure. |
| Inheritance | Mapping and record inheritance, nested destination-local evaluation, child identity, policy lifetime, version movement, history ordering, sensitivity, and retained-node budgets. |
| Projection/schema | Inline fields, recursive schemas, dynamic values, nil/empty values, cycles and limits, text codecs, secret states, callback ownership, and independent loads; preserve existing cloning behavior where shared descriptors/state are affected. |
| Facade and consumers | Forward conversion, detached records, supported API boundaries, source-form equivalence, maintained simple/catalog/discovery examples, and safe explanation/diff. |
Examples of established boundaries include
[source-form equivalence](../../facade_equivalence_test.go),
[discovery integration](../../facade_discovery_test.go),
[final identity validation](../../facade_final_sequence_identity_test.go),
[inheritance history](../../internal/engine/inheritance_history_test.go), and
[projection and cloning](../../internal/engine/projection_test.go).
Use applicable race checks when changing ownership or shared state. Reuse checks
for unchanged content and keep benchmark work separate unless new evidence
justifies it and it is authorized.
Completion requires reviewable evidence that:
- Simple and advanced consumer examples remain concise and valid, with no new
public concepts required for existing workflows.
- Both load entry points reach one normalized internal model; acquisition no
longer reconstructs broad options or uses version state to transport defaults.
- Import, context, inheritance, and reflection responsibilities have identifiable
owners and explicit handoffs without duplicate semantic implementations.
- Unused reverse conversions and their exclusive adapters are removed, with
useful inspection ownership coverage retained.
- Contracted values, presence, provenance/history, diagnostics, sensitivity,
resource lifetime, and algorithmic corrections remain protected by appropriate
tests and source review.
- The reviewed scope is accounted for against all six audit findings, with clone
traversal review deferred to future work. Completed portions have validation
evidence rather than being closed because code moved.
- Consumer guidance, contributor references, and internal documentation describe
the resulting implementation. Current contracts continue to own behavior.
The [implementation plan](implementation.md) defines private records, file owners,
and stage boundaries. No open scope questions remain, and no additional feature
semantics are needed to carry out the selected refactors.
## Settled Decisions
### OQ-1: Should The Public Loading APIs Change In This Effort?
**Settled:** retain `Load`/`Options`, `LoadLayered`/`LayeredOptions`, `Input`, and
`Underlay`, including their existing overlay-context behavior. Consolidate their
private inputs, conversions, and acquisition machinery, and improve consumer
guidance. The [consumer-facing target](#consumer-facing-target) records the
rationale and tradeoffs. This decision governs the consolidation effort; it does
not establish a perpetual restriction on future API design.
### OQ-2: How Much Clone Cleanup Should This Effort Commit To?
**Settled:** defer clone traversal review and any resulting cleanup to
[future.md](future.md#4-clone-traversal-review). Neither the review nor clone
refactoring is part of this consolidation effort. Projection and schema
refactoring remain in scope, including protection of existing cloning behavior
where they share descriptors or operation state. The future review can conclude
that no clone edits are warranted; its goal is demonstrated maintenance benefit.

View File

@@ -83,3 +83,36 @@ evidence are preserved in Git history at `8e91a2b` under `docs/roadmap/`.
across loads without introducing aliasing?
- What cache lifetime and concurrency design would make schema reuse worthwhile
without excessive retained memory?
## 4. Clone traversal review
Status: deferred. Review typed configuration cloning after the projection and
schema consolidation work, when the resulting structure can inform the review.
This is future maintenance work, not a prerequisite for completing consolidation.
The complexity audit identified `valueOperation.clone` in
[clone.go](../../internal/engine/clone.go) as a candidate for cleanup. Much of its
size is straightforward type dispatch. Focused handlers may clarify allocation,
recursive copying, cycle tracking, secret-state handling, or text-codec
reconstruction, but extraction may also add indirection without making the code
easier to understand.
When this work is taken up, assess the traversal and record whether changes are
worthwhile; a conclusion that no refactor is needed is valid. Make focused
changes only where the maintenance benefit justifies the code movement. Keep
cloning's typed-copy semantics distinct from projection's document-building
semantics. Preserve detached values, nil/empty distinctions, physical inline
fields, cycle/depth checks, and unresolved versus validated secret handling.
Reuse existing ownership, codec, secret, and traversal tests, adding coverage for
consequential gaps. No performance improvement is assumed.
### Questions to resolve before implementation
- After projection and schema consolidation, which clone branches still mix
responsibilities enough to benefit from extraction?
- Would a proposed helper keep allocation, recursive copying, and cycle cleanup
easier to follow together, or merely move a short loop out of sight?
- Are any small shared operations semantically identical across traversals, or
would sharing require special cases that obscure their different purposes?
- Which ownership or special-value risks, if any, lack sufficient existing test
coverage for the proposed changes?

View File

@@ -0,0 +1,783 @@
# Consolidation Implementation Plan
Status: complete. Stages 1 through 20 are complete.
The target executor is a Luna coding agent working through one end-to-end sprint.
## Authority, Baseline, And Execution
The [consolidation roadmap](consolidation.md) owns the target state, rationale,
and settled scope. Start from `8a99e6107001c69117a41dc37bc6ea09d2910af9` plus the
commit introducing this plan. Production code is unchanged from the audited
`5959537d1adc2516e7f7e8881df72b707ec48c3f`; intervening commits consolidated the
planning documents and settled the public-API and clone decisions.
Read [development.md](../development.md), [agents.md](../policy/agents.md),
[architecture.md](../policy/architecture.md), [testing.md](../policy/testing.md),
and [documentation.md](../policy/documentation.md) as applicable. Use the code
graph for discovery and confirm source at the implementation baseline. Current
public comments and [config.md](../config.md) define behavior; this plan does not
add public obligations.
Implement only the selected consolidation. Retain the public `Load`,
`LoadLayered`, options, inputs, callbacks, source/search descriptions, inspection,
and sequence-registration APIs. Keep the root-to-engine-to-document/YAML
dependency direction. Clone traversal review/refactoring remains deferred under
[future.md](future.md#4-clone-traversal-review). Changes to shared descriptors must
preserve existing cloning behavior. This sprint does not publish a release,
update dependencies/toolchain requirements, change configuration semantics, or
introduce complexity/performance gates.
Stages are bounded work assignments, not independently shippable commits.
Intermediate stages may leave callers, tests, or compilation unfinished until a
later stage. Proceed directly through those dependencies; do not build temporary
compatibility layers solely to keep each stage compiling. Update affected tests
with their callers, and run the focused checks below when that portion builds.
Record any deferred verification accurately. The completed sprint must compile,
pass its checks, and leave no superseded plumbing. Commit completed, reviewable
units under the agent policy; stage boundaries do not require separate commits.
Preserve existing meaningful validation/processing order, diagnostics, ownership,
and limits. Private helper order and representation are not new contracts. Do
not add rejection rules, error recovery, caches, new merge strategies, another
pipeline framework, or tests for incidental helper choreography. A correctness
issue discovered during the refactor needs identifiable regression evidence and
an explicit explanation, rather than an unexplained expectation change.
## Chosen Private Design
Use the existing `loadRequest[T]` as the normalized complete-load model and
`loadInput` for each selected input. Keep generic application definitions and
the final lifecycle in `executeLoadRequest[T]`; acquisition is non-generic.
The following records are ordinary engine-private data, not extension points.
Use these names to make migrations concrete; their fields describe the selected
ownership model, not a requirement to duplicate data already available there.
| Record | Fields and ownership |
| --- | --- |
| `acquisitionPolicy` | Read-only operation snapshot of `importDefaults`, `sequencePolicy`, and `importLimits`. Snapshot caller-owned defaults at entry, preserving nil versus empty. |
| `acquisitionState` | Borrowed pointers to the load's `importAccounting` and `sourceVersionState`. One instance per load; version state no longer transports import defaults. |
| `tierAcquisitionState` | The tier's visited-resource map and shared host namespace token. Underlay groups each own a fresh instance. Existing group graph aggregation and namespace anchors retain their current owners. |
| `inputAcquisitionRequest` | `input loadInput`, source-version role, root-directive context, applicable protection rule, deferred-assembly choice, and `ordinaryOverlay string`. The last field carries only the ordinary main input's overlay-dependent validation/context timing; it is empty for underlays and layered main. |
| `importAcquisitionRequest` | The input request, parsed main source, and already computed source-version/directive processing result. It preserves the distinction between source-local processing and application-path protection. |
| `resolvedFilesystemContext` | `native *nativeNamespace` (nil means host), `hostToken *namespaceToken` where supplied, and `rootPath`, `basePath`, `mainPath` strings. Native facts use slash-relative paths; host facts use the existing canonical host paths. The record contains no open root/file or acquisition access object. |
The main private entry is
`acquireInput(request inputAcquisitionRequest, policy acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (assembledInput, []Diagnostic)`.
Import acquisition uses the corresponding named request with the same borrowed
policy/state/tier owners. An access-based worker receives a resolved filesystem
description and an explicitly owned `*filesystemAccess`, rather than more
positional paths and independent optional policy parameters. Retain
`assembledInput`, `acquisitionScope`, and `acquiredContribution` as the structural
handoffs; add an optional resolved-context field to `assembledInput` for reuse.
`importTraversal` borrows the policy and state references. Active routes, stack,
current item scope/prefix/fallback, and the acquired inclusion graph remain
traversal-local. Nested item traversal shares the active graph and tier repetition
state; it does not create another tier. Reusing a file in a later tier remains
valid. Do not collapse namespace identity, lexical candidate spelling, and stable
scheduling keys into one key.
Context facts are obtained at the existing points where those facts are needed.
There is no global eager context-validation pass or persistent live-handle owner.
Directory checks, canonicalization, handle opening/closing, and passive view
creation remain distinguishable. Reusing a description does not authorize
earlier checks that bypass an existing earlier error or implicit loading activity
after publication. Consumer-initiated passive filesystem operations retain their
existing contract.
For inheritance, retain both scope adapters and one executor. Use private
prepared-child/prepared-parent records for combination inputs, then an
`inheritanceTemplateResult` containing the resolved value, origins, local policy,
policy templates, and the controls/transfers/replacements/operations produced by
that template. Sequence position evidence lives in the existing operation
records. Merge callbacks mutate that local result, not the enclosing executor's
accumulator. The executor appends the result and applies existing accounting.
For reflection, keep `valueOperation` and `schemaBuilder` as their existing state
owners. Extract substantial value-kind and inline-field operations as methods;
do not introduce a generic reflection visitor or merge the two schema views.
## Validation Groups
Use the installed compatible Go toolchain and already acquired dependencies as
described in the development guide. Commands run from the repository root with
`GOTOOLCHAIN=local` and `GOPROXY=off`. These are development checks for this
refactor, not additions to the release procedure.
| Group | Commands and purpose |
| --- | --- |
| F: facade cleanup | `go test . -run 'SourceInfo|SourceVersionCheck|ImportSelection|Facade|PublicFacadeBoundary|DependencyImportBoundary'` checks useful forward evidence and the existing public boundary. |
| A: acquisition integration | `go test ./internal/engine -run 'Load|Grouped|Source|Search|Discovery|Import|Glob|Filesystem|Native|Destination|Overlay|Resource|Version|Sequence|Collection'`; `go test . -run 'Facade|EquivalentSourceForms|Search|SourceContext|ImportDefaults|SourceVersion'`; `go test ./examples/consumers`. Covers normalization, ordinary/layered acquisition, selection, contexts, scopes, and consumer workflows. |
| I: inheritance | `go test ./internal/engine -run 'Inheritance|Inherited|Sequence|Collection|Version|History|Explain|Sensitivity|Secret'`; `go test . -run 'Inheritance|Inherited|SequenceIdentity|FinalSequence'`. Covers shared adapters and their evidence. |
| R: reflection | `go test ./internal/engine -run 'Schema|Decode|Projection|Clone|Inline|CompareDocument|Callback|Ownership|Independent|Secret'`. Covers descriptors, projection, shared cloning behavior, and ownership. |
| Final | `./scripts/check.sh` followed by `go test -race ./...`. The first owns formatting, whitespace, ordinary tests/examples/doc links, vet, and build; race validation is appropriate because state and ownership handoffs change throughout this sprint. |
Run F after Stage 2, A after Stages 9 and 13, I after Stage 16, and R after
Stage 18 when buildable. If earlier migrations still prevent compilation, repair
them at the next integration point and run the deferred group there. Run the
final checks once on the finished code/documentation; repeat only for subsequent
changes affecting their evidence. No benchmark campaign, timing threshold,
traversal-count test, extended fuzz campaign, CI wait, or manual platform check is
required by this plan. Add narrow behavior tests only for consequential gaps
identified below that existing tests do not already cover.
## Stage 1 — Establish The Starting Map (Complete)
**Work:**
1. Confirm the revision/worktree and preserve unrelated work. Read the roadmap's
settled decisions and the selected owners above.
2. Map the current callers of `selectOptions`, `assembleBaseInput...`, both
`loadImported...` ladders, `contextsForAcquiredSources`, the selected-overlay
loaders, and `applyInheritanceScopeTemplates`. Inspect test callers as well
as production callers; some old entry variants exist mainly for tests.
3. Identify the existing behavioral owners for repeated resources within/across
tiers, enabled/disabled version checking, selected/absent searches, ordinary
versus independent layered overlays, item-local helper scopes, inherited
identity/history, inline fields, and detached records.
4. Keep brief working notes or implementation progress in this plan. Reuse
existing validation evidence where applicable; establish an implicated
baseline only when needed to distinguish a pre-existing failure. Do not
generate another broad audit or blanket characterization-test suite.
**Finish:** a concrete caller/coverage map and any actual coverage gaps. No
production change or empty commit is needed for orientation alone.
Progress notes (2026-10-25): the worktree was clean at `8ab1568`, immediately
after the planning baseline `8a99e61`; the audited production baseline
`5959537` is present. `selectOptions` is called by the ordinary and grouped
load entry points. The `assembleBaseInput...` wrappers converge on
`assembleBaseInputTrackedForRole`, reached from ordinary loading, layered main
loading, and underlay-group acquisition. The tracked import wrappers converge
on `loadImportedDocumentsTrackedWithProcessing`; production callers are input
acquisition and underlay-group acquisition, while older convenience variants
also have direct test callers. `contextsForAcquiredSources` is shared by ordinary
loading, layered assembly, input acquisition, and overlay application. Selected
overlay loading is reached from ordinary overlay application and layered input
assembly. Both inheritance adapters, for ordinary and identified sequences,
feed `applyInheritanceScopeTemplates`.
Existing behavioral owners cover repeated/cyclic resources and cross-tier reuse
(`internal/engine/imports_test.go`, `facade_equivalence_test.go`); default and
enabled source-version modes (`internal/engine/source_version_test.go`,
`facade_source_version_mode_test.go`); selected/absent discovery
(`internal/engine/discovery_load_test.go`, `facade_discovery_test.go`); ordinary
and layered overlays (`internal/engine/selected_overlay_test.go`,
`facade_layered_test.go`); item-local helper scopes (`internal/engine/imports_test.go`,
`facade_import_lifecycle_test.go`); inheritance identity/history
(`internal/engine/inheritance_identity_history_test.go`,
`internal/engine/inheritance_history_test.go`); inline fields
(`internal/engine/schema_test.go`, `internal/engine/projection_test.go`); and
detached results and callback ownership (`internal/engine/metadata_test.go`,
`internal/engine/callback_ownership_test.go`, `facade_diff_test.go`). No specific
coverage gap surfaced in this map; later focused checks remain assigned in the
validation groups above.
## Stage 2 — Remove Unused Reverse Bridges And Share Real Facade Conversion (Complete)
**Owners:** root `conversion.go`, `records_test.go`, `lifecycle.go`, `layered.go`,
and engine `records.go`.
1. Reconfirm callers, then delete `toEngineSourceInfo`, `toEngineOrigin`,
`toEngineSourceVersionCheck`, and `toEngineImportSelections`, plus
`SourceInfoWithPlacement`, `SourceInfoWithImportSelections`,
`importSelectorFromString`, and `importSelectionOutcomeFromString` when their
only remaining purpose is the deleted chain.
2. Rewrite the three reverse-round-trip record tests using genuine engine output
and forward conversion/public inspection. Retain placement, version-evidence,
selection-evidence, nil/empty, and detached-copy protection. Engine test
fixtures can use existing loading APIs; do not add production fixture setters.
3. Extract one root `toEngineOverrides` helper and use it from both option
converters. Reuse `toEngineInput`, filesystem, path, definition, and default
conversion where already shared. Preserve slice ownership and nil/empty
behavior; do not make conversion interpret/validate input values.
4. Retain reverse diagnostic conversions used by `WriteDiagnostics`, forward
source/origin inspection conversion, and all public named types.
**Finish:** the unnecessary cross-package inverse interface is gone and real
conversion is shared without a new transport layer. Run F.
Progress notes (2026-10-25): removed the unused reverse source-record bridge
and engine setters, replaced synthetic round-trip fixtures with engine loads,
and shared override conversion across ordinary and layered options. Validation
group F passed with `GOTOOLCHAIN=local GOPROXY=off go test . -run
'SourceInfo|SourceVersionCheck|ImportSelection|Facade|PublicFacadeBoundary|DependencyImportBoundary' -count=1`.
## Stage 3 — Normalize Preflight Without Fabricated Options (Complete)
**Owners:** engine `options.go`, ordinary entry in `load.go`, grouped entry in
`load_request.go`, and their option tests.
1. Extract non-generic input normalization/validation over `loadInput`, returning
the normalized input plus diagnostics. Pass the ordinary overlay name only
where its context requirements apply. Keep discovery-description validation,
placement checks, normalized destinations, and filesystem option checks at
their existing pre-acquisition boundary.
2. Separate load-wide import-default validation, version-rule/mode validation,
and sequence-registration compilation from per-input checks. Share their
mechanisms between entries without adding a second interpretation of options.
3. Keep the public/engine entry arity handling and each entry's existing error
classification. Ordinary loading still translates its one selected options
value; layered loading validates main, all underlay inputs, and its independent
overlay context in the existing meaningful order. Preserve canonical diagnostic
accumulation and early returns, including invalid options before reads.
4. Replace the grouped entry's constructed `Options[T]` values with direct
normalization of `loadInput`; set the effective protection rule from the
normalized main placement. Leave typed application behavior in the definition.
5. Remove `importDefaultsValidated` and preflight-only forwarding machinery when
callers migrate. Update tests through the entry/helper that owns the behavior,
rather than keeping `selectOptions` solely as a test compatibility surface.
**Finish:** both entries produce validated `loadRequest[T]` values without using
the ordinary options type to validate grouped inputs. Acquisition may still use
old options until Stage 6; do not broaden this stage into that migration.
Progress notes (2026-10-25): added shared normalization over `loadInput`, moved
load-wide defaults, source-version, and sequence-registration validation into
shared settings preparation, and removed `selectOptions` plus its validation
flag from grouped preflight. `GOTOOLCHAIN=local GOPROXY=off go test ./...` passed.
## Stage 4 — Express Input Acquisition With Non-Generic Requests (Complete)
**Owners:** new engine `acquisition.go` and existing `load_input.go`.
1. Define `acquisitionPolicy`, `acquisitionState`, `tierAcquisitionState`,
`inputAcquisitionRequest`, and `importAcquisitionRequest` as selected above.
Reuse existing enums, policy records, and accounting; introduce no new public
controls or duplicate budget counter.
2. Replace the generic `assembleBaseInputTrackedForRole` implementation with
`acquireInput`. Translate its references to source, imports, filesystem,
placement, role, directive context, version rule, policy, and deferred
assembly into explicit request/policy/state fields.
3. Keep its phases visible: search selection; parse and failed-source evidence;
disabled-directive checks; source-local version/directive processing; import
acquisition or single-input placement; accounting/repetition registration;
context handoff. Small helpers may isolate version/directive preparation.
4. Preserve discovery resolution once per input consumption and the acquired
concrete source on both success and failure. Keep mapping diagnostic
destination projection and the ordinary-overlay context deferral unchanged.
5. Retain `assembledInput`'s existing structural/evidence fields. Do not merge
parsed/local payloads with placed documents or eagerly compose deferred input.
**Finish:** the shared input-acquisition implementation no longer depends on
`Options[T]` or an application type. Call-site and import-worker migration is
completed by Stages 5–6; temporary non-compilation is acceptable.
Progress notes (2026-10-25): introduced the non-generic acquisition request and
policy/state/tier records, moved the input-acquisition implementation into
`acquireInput`, and added the resolved-context handoff field. The focused engine
test command currently cannot compile because the old assembly callers in
`load_input.go` and `load_request.go` are scheduled to migrate in Stage 6; test
verification is deferred until that integration.
## Stage 5 — Consolidate Import Requests And Default Ownership (Complete)
**Owners:** engine import entry points, `importTraversal`, `source_version.go`,
and tests that directly invoke the old import variants.
1. Replace the `loadImportedDocumentsTracked...` ladder with
`acquireImportedInput` using `importAcquisitionRequest` and the shared
policy/state/tier owners. Replace the access-based ladder with one worker
receiving the same request plus explicit access/resolved facts. Keep only
genuinely distinct convenience entries, not one wrapper per optional field.
2. Move `importDefaults` out of `sourceVersionState`. Resolve declarations from
`acquisitionPolicy.importDefaults`, including nested item scopes. Import
defaults must work independently of whether a version rule/state is active.
Preserve enabled version processing and its separate local/effective rules.
3. Make traversal references to aggregate accounting, version evidence, tier
repetition, and immutable policies point to their existing operation owners.
Keep active-route tracking, scope switching, and inclusion edges local to
the traversal. Do not allocate a fresh visited map inside a nested item scope.
4. Preserve nil version-state behavior used by lower-level operations and the
existing default processing branch. Replace optional positional/variadic
policy passing with explicit inputs for meaningful operation distinctions.
Private test entry spellings do not need compatibility wrappers; preserve
their relevant behavioral cases through the resulting operation.
5. Migrate direct import tests to named inputs. Small test-only constructors may
supply default state/policy values when this reduces repetitive fixtures.
Keep their behavior assertions intact.
**Finish:** the import request surface has named cohesive inputs; defaults no
longer travel through version processing; no forwarding ladder remains merely
to supply another nil/default positional parameter.
Progress notes (2026-10-25): replaced the import entry ladders with named
requests and one access worker, moved defaults to acquisition policy, and
retargeted direct import tests. The focused engine test command cannot compile
yet because the remaining ordinary and grouped assembly callers are scheduled
for Stage 6; verification is deferred until that integration. `git diff --check`
passed.
## Stage 6 — Wire Ordinary And Grouped Lifecycle To Acquisition (Complete)
**Owners:** engine `load.go`, `load_request.go`, `load_input.go`, related tests.
1. Construct the load's acquisition policy and mutable state once in
`executeLoadRequest`. Keep the definition and schema/callback lifecycle there.
2. Call `acquireInput` for main using its ordinary/layered directive context,
main version role, current protection rule, and deferred assembly. Preserve
the existing schema-build and version-check timing, including the different
default/enabled source-version paths.
3. Have `acquireUnderlayGroupScopes` use the same load policy/accounting/version
state and one fresh tier state per group. Remove `Options[struct{}]`
construction. Preserve explicit namespace anchors and graph reanchoring.
4. Main acquisition still precedes underlays. Strict assembly still follows
the existing dependency-ready scopes and lower-tier identity information.
Keep group-local repetition separate from load-wide limits, and retain the
existing selected-overlay duplicate-source checks.
5. Replace the ordinary `applyInputLayers` options argument with the precise
main/overlay/protection/policy inputs it uses. A named overlay request will
complete that migration in Stage 9. Remove remaining production
`assembleBaseInput...` variants and the acquisition-only `sequencePolicy`,
`deferAssembly`, and `directiveContext` fields from `Options`. Retain the
placement, destination, and import-default fields used by facade translation.
6. Retain real lower-level test/benchmark capabilities with direct request
construction or test-only adapters. Do not delete meaningful source/limit
cases to make removed helpers disappear.
**Finish:** no executor or underlay caller reconstructs broad/generic options
for acquisition. The shared path is wired end to end; context and import-flow
extraction continue in later stages.
Progress notes (2026-10-25): `executeLoadRequest` now creates one acquisition
policy/state and acquires the main input through `acquireInput`; underlay groups
reuse those load owners with a fresh tier state. Removed the generic base-input
assembly variants and the acquisition-only fields from `Options`, and adapted
direct tests to construct named requests. A search-source regression was caught
and fixed during verification. The focused engine selection, facade selection,
consumer example tests, and `GOTOOLCHAIN=local GOPROXY=off go test ./...` passed.
## Stage 7 — Separate Resolved Filesystem Facts From Access (Complete)
**Owners:** new engine `filesystem_context.go`, `filesystem.go`,
`filesystem_access.go`.
1. Add `resolvedFilesystemContext` with the fields selected above. Reuse native
namespace/token identity; do not compare arbitrary `fs.FS` values or equate
equal labels with equal namespaces.
2. Extract host and native fact derivation from `importFilesystem` and
`nativeFilesystemAccess`. Prefer a selected `SourceInfo.Path` when it already
supplies canonical source identity. Preserve source-derived file bases,
explicit in-memory roots/bases, native slash paths, and the root-only absent
host default.
3. Keep separate functions for the directory checks and opening access that
currently occur at each operation. A returned description does not mean
every possible check has run. Do not add validation-state flags or an
all-purpose eager resolver with stronger acceptance rules.
4. Adapt the import worker to consume resolved facts and explicitly own its
access until the existing acquisition handoff closes it. Keep close-error
reporting and primary-error precedence unchanged. Native access borrows its
caller filesystem and must not close it.
5. Keep any old derivation adapters only while genuine callers migrate in
Stages 8–9; remove them at the end of that migration.
**Finish:** common context facts have one private representation while validation
timing and live resource ownership remain explicit.
Progress notes (2026-10-25): added shared host/native filesystem descriptions,
separated directory validation and access creation, and migrated import
acquisition to pass resolved facts alongside its owned access. The prior
derivation adapters remain for the validator and overlay migration. Focused
acquisition tests, `GOTOOLCHAIN=local GOPROXY=off go test ./...`, and
`git diff --check` passed.
## Stage 8 — Carry Context Facts Through Input And Passive Views (Complete)
**Owners:** engine `load_input.go`, `source_context.go`, `load_request.go`, and
the ordinary context backfill in `load.go`.
1. Add the optional resolved-context field to `assembledInput`. Populate it
from import acquisition or the existing non-import context validation point.
When context work is deferred for an ordinary overlay, leave it deferred.
2. Replace the source-kind derivation branches of `contextsForAcquiredSources`
with `contextsForResolvedSources`: consume resolved facts and source records,
construct the existing rooted host/native passive view, and compute each
source's relative base using the appropriate path rules.
3. Share relative-base/containment calculation where applicable; retain each
acquired source's own path and source-ID association. Do not substitute the
first source record for the selected main file or attach all values to main.
4. Migrate normal input acquisition, underlay contexts, and ordinary context
backfill. Resolve only still-missing facts at the same lifecycle point as
before. Retained descriptions/views never capture `filesystemAccess` or
`*os.Root`; the existing passive `ResourceContext` contract stays unchanged.
5. Retain failure evidence, root-only absent host context, discovered native
outcomes, and deferred search-base behavior. Update focused context tests
only where private function signatures changed.
**Finish:** ordinary/underlay acquisition and passive views use the common facts;
selected-overlay callers are the remaining migration in Stage 9.
Progress notes (2026-10-25): input acquisition now retains resolved filesystem
facts and builds passive contexts from those facts for ordinary and underlay
sources. Migrated the ordinary backfill and reordered-source context test. The
selected-overlay adapters remain for Stage 9. Focused context/acquisition tests,
`GOTOOLCHAIN=local GOPROXY=off go test ./...`, and `git diff --check` passed.
## Stage 9 — Migrate Selected Overlays Without Merging Their Semantics (Complete)
**Owners:** engine `selected_overlay.go`, ordinary `applyInputLayers`, and
`assembleLayeredInput`.
1. Define a private `selectedOverlayRequest` containing the overlay name,
resolved filesystem description, protection rule, sequence policy, and
existing limits/version state. Derive context at the caller's existing
lifecycle point. Pass assembly and source evidence explicitly to the worker.
2. Route ordinary overlays through the main source's context; route layered
overlays through their independent `OverlayFilesystem`. Use named operation
inputs instead of fabricating a `sourceBytes`/empty parsed-main pair merely
to select a context branch. Preserve the actual absent-host root behavior.
3. Return or retain the resolved description used by overlay acquisition and
build overlay passive context from it. Reuse main facts only when that
ordinary operation previously used the same root. Context facts do not share
a live handle between the main import traversal and the later overlay read.
4. Keep the production collection-aware overlay path. Migrate meaningful
lower-level tests to that path with explicit policy/limits, retaining their
budget, version, and failure assertions. Delete the test-only legacy branch
selected by missing variadic policy and its exclusive adapters, rather than
treating optional argument count as a contract. The baseline production entry
points all supply the policy argument.
5. Keep overlay source IDs, duplicate detection, version protection, directory
validation, read/close errors, aggregate accounting, and post-read application
order. Remove superseded context derivation and overlay-forwarding adapters.
**Finish:** all acquisition/context callers share facts and named inputs while
ordinary and independent layered overlay semantics remain distinct. Run A when
buildable; carry any genuine remaining migration dependencies to the next
integration opportunity instead of adding scaffolding or stopping the sprint.
Progress notes (2026-10-25): introduced the named overlay request and migrated
ordinary and independent layered overlay acquisition to resolved facts plus
explicit access. Replaced the variadic compatibility branch with the existing
collection-aware path, removed old context adapters, and retained overlay
validation timing, source evidence, budgets, version protection, and close
reporting. Validation group A, consumer examples, the full Go suite, and
`git diff --check` passed.
## Stage 10 — Give Import Responsibilities Focused Files (Complete)
**Owners:** engine `imports.go` and new `import_acquisition.go`,
`import_scope.go`, `import_projection.go`.
1. Put the request entry/access worker and traversal admission/orchestration in
`import_acquisition.go`. Keep `imports.go` as the small common owner for
limits, accounting/route/traversal data, and shared import diagnostics.
2. Move `composeAcquiredInputWithPolicies`, `composeAcquiredScopeWithPolicy`,
pending-item composition, placement reservation, and their item-scope
assembly helpers into `import_scope.go`.
3. Move nested-origin/policy/evidence projection, acquired source destinations,
version-evidence projection, sequence-prefix helpers, and diagnostic-path
projection into `import_projection.go`. Scheduling and graph identity remain
with their existing graph/scheduler owners.
4. Make the acquisition-to-closed-resources-to-assembly handoff visible. Keep
evidence projection based on existing composition applications/position maps;
file movement does not authorize another merge, full-tree replay, or scan of
every unrelated policy for each record.
**Finish:** maintainers can locate acquisition, local strict assembly, and evidence
work separately. Keep this stage structural; parse/admission extraction follows.
Progress notes (2026-10-25): moved request acquisition and traversal admission to
`import_acquisition.go`, local strict composition and item-scope assembly to
`import_scope.go`, and origin, destination, version-evidence, and diagnostic-path
projection to `import_projection.go`. Shared limits, traversal data, and import
diagnostics remain in `imports.go`. `./scripts/check.sh` passed.
## Stage 11 — Extract Import Declaration Parsing Operations (Complete)
**Owner:** engine `import_spec.go`.
1. Retain the outer mapping check, exactly-one-selector check, source-order field
walk, applicability checks, and final selector parsing in
`parseImportDeclarationSpec`.
2. Extract boolean assignment, wildcard-set parsing/validation (shared by
`pattern` and `exclude`), placement parsing, and selector-value parsing into
ordinary private helpers. Reuse existing `parseStringSet`, authored-member
sites, and destination/depth parsers.
3. Preserve unspecified pointers and set flags, especially nil versus empty
pattern lists. Keep error paths/locations tied to the authored selector or
list member. Do not reorder consequential checks or replace the selector
matrix with a configurable parser registry.
**Finish:** field decoding is readable without changing declaration syntax,
selector rules, default resolution, or diagnostics. Keep behavior tests as the
owner; no helper-by-helper assertion inventory is needed.
Progress notes (2026-10-25): extracted boolean assignment, shared wildcard set
assignment, placement parsing, and selector-value parsing into private helpers.
The declaration parser retains its validation ordering, authored member sites,
and nil-versus-empty presence behavior. Import-focused tests and
`./scripts/check.sh` passed.
## Stage 12 — Extract Directory Entry Admission And Descent (Complete)
**Owner:** engine `import_selection.go`, using existing directory/link helpers.
1. Keep root normalization/deduplication, root ordering, the explicit traversal
stack, candidate union, sorting, exclusion, and required-empty decision visible
in `selectDirectoryImportCandidates`.
2. Extract coherent entry inspection and descent preparation. Return a small
result identifying skip, admitted file, or directory frame to enter, plus a
diagnostic when appropriate. Keep stack/active-directory mutation with the
traversal so its lifetime is visible; do not invent a general visitor API.
3. Preserve hidden/follow/depth behavior, distinct lexical candidates versus
canonical access identity, resource-entry accounting, root-error handling,
and active-path cycle detection. A later non-active alias remains eligible;
exclusions still do not suppress discovery errors.
4. Keep globs and first-match main-file discovery on their existing mechanisms.
Share a helper only where its existing admission semantics actually coincide.
**Finish:** substantial entry/descent policy is separated from stack orchestration
without changing the selection algorithm or adding repeated root walks.
Progress notes (2026-10-25): extracted entry inspection and file/directory
admission into a result-returning helper. The selector retains root handling,
stack and active-path mutation, candidate union, sorting, exclusions, and the
required-empty decision. Directory-focused tests and `./scripts/check.sh` passed.
## Stage 13 — Extract Imported-Source Admission From Recursion (Complete)
**Owner:** engine `import_acquisition.go` after Stage 10.
1. Keep `importTraversal.visitPath` as the visible coordinator of containment,
canonical identity, cycle/repetition checks, read/parse, directive/version
processing, placement, and recursive declaration traversal.
2. Extract source admission/read-and-process results into a private
`admittedImport` carrying canonical resource/node identity, source record,
processed payload/declarations, and processing evidence. Return an explicit
optional-missing outcome; do not represent a read failure as absence.
3. Preserve when source records are appended, rolled back for optional missing
input, and retained for failures; preserve source IDs, accounting increments,
version-callback ordering, and inclusion-edge creation on repetition/cycles.
4. Pass the admitted record to mapping/item placement helpers rather than
repeating its payload, declarations, and processing evidence as parallel
arguments to `visitItemSource`.
5. Factor route push/pop and item-scope save/restore only where it makes their
shared lifetime clearer. Keep active graph state across item scopes and
reserve placement nodes once. Item helpers complete before outer placement;
they are strict local peers, not a precedence layer.
**Finish:** admission decisions and recursion have clear handoffs, with one
read/process path and the original scheduling/evidence algorithms. Run A again
because Stages 10–13 changed the import implementation after the first checkpoint.
Progress notes (2026-10-25): introduced an explicit admitted-import record and
optional-missing outcome for source read/parse/version processing. `visitPath`
retains containment, identity/cycle checks, graph updates, and recursive
placement coordination; item placement receives the admitted record directly.
Validation group A and `./scripts/check.sh` passed.
## Stage 14 — Extract Inheritance Child And Parent Preparation (Complete)
**Owners:** engine `inheritance_schedule.go`, new `inheritance_prepare.go`, and
existing scope/policy helpers.
1. Extract preparation of the child's directive-free payload, direct origins,
local policy templates, and optional control/control-origin evidence.
2. Extract parent preparation: clone/project policies, lower them against the
child's claims, exclude parent identity from the inherited payload, and
prepare the separate parent history snapshot where currently required.
Keep effective policy templates and transfer construction available to the
combination result rather than reconstructing them afterward.
3. Use focused prepared-child/prepared-parent records of these related values.
Borrow authored/resolved parent inputs read-only; retain existing clone points
for mutable payloads and history. Keep the current preliminary lower-bound
budget decisions at the executor's existing point before materialization.
4. Keep mapping and identified-sequence adapters, the stable Kahn scheduler, and
the parentless/cancelled path semantics intact. Do not flatten nested directives
before their destination-local evaluation.
**Finish:** the executor no longer embeds child/control and parent/policy
preparation. Actual recursive combination/evidence moves in Stage 15.
Progress notes (2026-10-25): extracted child/control and parent/policy preparation
into focused records in `inheritance_prepare.go`. The executor retains its
preliminary budget checks, scheduler, and combination path, while history and
transfer inputs stay available to combination. Engine inheritance coverage and
facade inheritance tests passed.
## Stage 15 — Return One Inheritance Combination Result (Complete)
**Owners:** new engine `inheritance_combine.go`, `inheritance_collection.go`,
and the shared executor.
1. Add `inheritanceTemplateResult` as described above and extract the existing
parent/child combination into a helper receiving prepared inputs and the
application policy. Parentless templates return the same result shape.
2. Keep `mergeRecursiveMappings` and `combineSequence` as the combination
mechanisms. Their callbacks update local origins/policy and local replacement
evidence. Have `applyInheritanceSequenceOrigins` return its replacement slice
instead of taking the whole enclosing scope result; append that slice to the
local template result while retaining its updates to the supplied origin maps.
3. Derive sequence history, transferred origins, retired/replaced policies,
identity preservation, and the transfer operation from that same combination.
Preserve `historyParent` and all policy/identity fields on the existing
`inheritanceOperation`; do not rerun a merge or project values to recover them.
4. Keep sequence validation, diagnostic canonicalization, location assignment,
replacement order, and no-parent cloning behavior at their established
semantic points. Keep child identity absent from inherited-parent history but
present with its own origin in the effective child.
**Finish:** merge callbacks no longer mutate the executor accumulator. The
resolved template and all evidence travel together for Stage 16 integration.
Progress notes (2026-10-25): added `inheritanceTemplateResult` and moved the
parent/child merge and evidence construction into `combineInheritanceTemplate`.
Callbacks now update the local origins, policies, and replacement evidence;
sequence-origin handling returns replacement records to the template result.
Parentless templates use the same result shape. Engine and facade inheritance
tests passed.
## Stage 16 — Finish The Inheritance Executor And Accounting (Complete)
**Owner:** engine `inheritance_schedule.go` and focused finalization helpers.
1. Reduce the executor to preparation, budget precheck, combination, accumulation,
and final rebuild. Append each template's control before its transfer/history
in the same order as before, and update the indexed resolved value/origin/
policy/template state once per template.
2. Extract retained-history charging for one template's returned operations and
replacements, reusing the same `retainedNodeCounter` across the scope as
before. Preserve pointer-based shared-node accounting, lower-bound checks,
scope-slot overhead, final expansion check, limits, and diagnostics.
3. Extract final rebuilding and scope policy/origin projection where it clarifies
the coordinator. Reuse `newInheritancePolicyIndex` once for scope localization;
do not return to scanning all policy entries for every slot.
4. Confirm both adapters and nested walking consume the same output. Retain
source-version movement, sensitivity, independent snapshots, assembled order,
cancellation, and policy retirement on actual replacement.
**Finish:** one readable executor coordinates explicit value/policy/evidence
handoffs, with no parallel inheritance implementation. Run I.
Progress notes (2026-10-25): extracted per-template retained-history charging and
scope rebuild/origin/policy projection into `inheritance_finalize.go`. The shared
counter, budget checks, final expansion check, and one policy index remain intact.
Both mapping and identified-sequence adapters still use the shared executor.
Validation group I passed.
## Stage 17 — Extract Substantial Projection Handlers (Complete)
**Owners:** engine `projection.go`, optionally new `projection_collection.go`.
1. Keep `projectConfiguration`'s schema/type guard and one `valueOperation`.
Keep `project`'s depth check, pointer/dynamic dispatch, and scalar conversion
choices visible; delegate structs, maps, and sequences to named methods.
2. Move struct member/inline flattening, collision checks, mapping-key charging,
and flattened-field/inline-map secret classification into `projectStruct`.
Preserve declaration order and the current decrement for a flattened
container that is not retained in the output.
3. Move deterministic sorted-map traversal into `projectMap`; keep its cycle
entry active until all recursive children complete. Move slice/array
construction into `projectSequence`, preserving nil-slice handling and the
slice-only cycle guard.
4. All recursive children call the same operation's dispatcher. Avoid duplicate
node charging/depth increments or early cycle release introduced by extraction.
Preserve float/text/duration/secret scalar handling and error paths.
**Finish:** dispatch and substantial handlers are separately readable, using the
original operation state. Do not refactor `clone`, `decodeValue`, or
`valueBuilder.build` as an additional cleanup project.
Progress notes (2026-10-25): delegated struct, map, and sequence projection to
methods on the existing `valueOperation`. The dispatcher retains depth checks,
pointer/dynamic handling, and scalar conversion; struct order, map sorting,
flattened-node charging, nil slices, and cycle guards remain in their original
paths. Validation group R passed.
## Stage 18 — Separate Schema Field Discovery And Assembly (Complete)
**Owner:** engine `schema.go`, optionally new `schema_fields.go`.
1. Keep `describeFields` as declaration-order enumeration, exported/ignored/tag
decisions, and dispatch to ordinary or inline handling.
2. Extract inline pointer-chain inspection and inline-struct/map assembly,
retaining recursion/building-state checks and the existing codec-versus-
structural distinction. Reuse the one builder/cache.
3. Make ordinary registration and inline flattening visibly maintain both
`members` and flattened `fields`/`fieldByName`/`inlineMap`. Preserve index
paths, nested secret propagation, ambiguity paths, and the single-inline-map
rule. Share registration helpers only for genuinely identical updates.
4. Preserve descriptor reuse and the physical-member view used by typed cloning.
Do not unify these views or add alternate field/tag policy. Existing schema,
projection, decoding, and cloning expectations remain the behavior baseline.
Progress notes (2026-10-25): kept declaration-order enumeration, exported and
ignored-field decisions, and tag parsing in `describeFields`; moved ordinary
registration, inline pointer inspection, and inline struct/map assembly into
focused helpers in `schema_fields.go`. Physical members remain distinct from
flattened fields and inline-map metadata. Validation group R passed.
**Finish:** field discovery and descriptor assembly have explicit responsibilities
without a new registration framework. Run R for both Stages 17–18.
## Stage 19 — Update Consumer And Internal Navigation (Complete)
**Owners:** `docs/consumers/README.md`, `docs/internal/overview.md`,
`docs/internal/imports.md`, and affected links/test-owner entries in
`docs/development.md`.
1. Add a concise consumer choice guide: ordinary `Load` for one selected source,
explicit discovery as a source usable by either entry, and `LoadLayered` for
underlay groups. Distinguish input-owned placement/import/filesystem choices
from load-wide definition/default/version/override choices and independent
layered overlay context. Link current API/config owners for exact rules.
2. Link the existing simple README example, bundled-file/layered examples,
import-focused prompt catalog, and discovered sequence catalog. Keep the
README's simple path simple; do not duplicate complete examples or rewrite
consumer domain rules to showcase new internal types.
3. Update implementation ownership and flow for normalized requests, acquisition
policy/state/tier lifetimes, resolved context versus live handles, import file
owners, and explicit inheritance/projection/schema handoffs. Fix moved-symbol
and behavioral-test navigation and stale comments affected by the refactor.
4. Preserve public contract definitions and accepted ADR decisions. Keep deferred
clone work in `future.md`. No policy change or new ADR is needed for the
selected private refactor.
Progress notes (2026-10-25): added a consumer entry-point and ownership guide,
linked the existing adoption examples, and documented normalized requests,
acquisition/context lifetimes, import owners, and inheritance/projection/schema
handoffs. Updated test navigation and retained clone review as deferred.
Documentation links and repository boundaries passed.
**Finish:** current documentation describes implemented structure, with the
roadmap/plan retaining future work and progress until release cleanup.
## Stage 20 — Verify And Close The Combined Sprint (Complete)
1. Search for obsolete import/input forwarding ladders, synthetic `Options`
acquisition construction, `sourceVersionState.importDefaults`, discarded
context derivation branches, unused inverse bridges, and temporary scaffolds.
Remove leftovers along with exclusive test scaffolding. Preserve distinct
operations and real diagnostic conversions; do not delete private code solely
because it resembles another helper.
2. Review the combined call flow: one normalization model, non-generic acquisition,
load versus tier versus traversal state, library-owned acquisition handles
closed before composition/publication, one combination outcome for inheritance
evidence, and one state owner per reflective operation. Review for added whole-tree
traversals, policy scans, or ownership copies without requiring measurements.
3. Account for every selected finding using the table below. Confirm any new
handoff coverage addresses a concrete gap. Existing correctness/algorithmic
regressions, public-boundary tests, and meaningful assertions stay effective.
4. Run the Final validation group, resolve failures, inspect the staged diff and
workspace, and commit the completed remaining work with plain-English
messages. Report changes, checks, limitations, and any unrelated work retained.
Do not tag, push, publish, or update consumer repositories in this sprint.
Progress notes (2026-10-25): reviewed the combined request/acquisition/tier and
traversal flow, the inheritance result handoff, and the shared reflection state
owners. The named obsolete source-version field, synthetic acquisition-options
construction, and inverse conversion symbols are absent; no additional cleanup
was needed. The final development check and race-enabled full test suite passed.
| Audit finding | Completion evidence |
| --- | --- |
| 1: imports | Stages 3–6 and 10–13 remove options reconstruction/default coupling and forwarding ladders; acquisition/scope/evidence owners and admission operations are clear. A and final checks pass. |
| 2: inheritance | Stages 14–16 return explicit template results, keep one executor, and preserve accounting and value/history/identity agreement. I and final checks pass. |
| 3: projection | Stage 17 extracts substantial handlers sharing operation state. R and final checks pass. Clone review remains deferred, not implicitly completed. |
| 4: schema | Stage 18 clarifies ordinary/inline assembly while preserving both descriptor views. R and final checks pass. |
| 5: contexts | Stages 7–9 share resolved facts across input/import/overlay/passive contexts without retaining handles or changing validation timing. A and final checks pass. |
| 6: facade | Stage 2 removes unused inverse conversions/adapters and shares real conversion. F and final boundary/consumer checks pass. |
**Finish:** the selected roadmap target is implemented, validated, documented,
and reviewable as one complete sprint. No unresolved product/scope decisions are
needed to begin these stages.

View File

@@ -0,0 +1,77 @@
# Consolidation Review Findings
Reviewed revision: `557e006a4ab98b666735a71b3845418c702f7cec`.
Comparison baseline: `8ab1568` (the committed implementation plan).
The review compared the [consolidation target](consolidation.md) and
[implementation plan](implementation.md) with the completed sprint, including
public/private conversions, preflight and acquisition, import selection and
evidence, filesystem contexts, inheritance, projection, schema construction,
changed tests, and documentation. The development check (`./scripts/check.sh`)
and full race suite (`go test -race ./...`) passed with Go 1.26.10 on Linux.
## 1. Complete The Non-Import Resolved-Context Handoff
Status: open. Priority: medium. Category: implementation gap and maintainability.
No user-facing correctness regression has been established for this finding.
**Target.** Roadmap internal target 2 and implementation Stages 7–9 select one
resolved filesystem description shared by validation/acquisition and passive
context construction. Stage 8 explicitly requires populating the description
at the existing non-import validation point and resolving only facts that are
still missing later.
**Evidence.** In [acquisition.go](../../internal/engine/acquisition.go),
`acquireInput` calls `validateFilesystemContext` at line 181 for inputs without
import traversal and with an applicable explicit context. That validator returns
only a diagnostic. Its host/native branches derive a resolved description but
discard it. At lines 220–226, acquisition sees `input.resolvedFilesystem == nil`
and calls `resolveFilesystemContext` again before building passive views.
For example, an ordinary `Load` of a byte or mapping source with explicit host
`FilesystemContext` and imports disabled resolves/canonicalizes the base and
root during validation, then derives those same facts again for its source
context. The retained description therefore does not complete the intended
validation-to-context handoff.
Additionally, [filesystem.go](../../internal/engine/filesystem.go) still contains
`validateAbsentHostFilesystemContext` (line 40), with its own root/base
canonicalization, root-defaulting, and containment logic. That derivation
duplicates `resolveAbsentFilesystemContext` in
[filesystem_context.go](../../internal/engine/filesystem_context.go) (line 29).
The two paths already maintain separate containment-diagnostic messages. Graph
caller inspection confirms that the old validator is reachable from
`acquireInput`; it is not unused migration scaffolding.
**Why address it.** The new model exists, but one important path still discards
and reconstructs its results, while absent-host context derivation has two
production implementations. This leaves part of audit finding 5 unresolved and
preserves the drift risk the consolidation was intended to remove. This is a
structural completion issue, not a claim of measured performance loss or a new
single-lookup guarantee.
**Required revision.**
- Have non-import context resolution/validation return or populate the resolved
description, and retain it on `assembledInput` at the existing validation
point. Later passive-view construction and ordinary-overlay handling should
reuse those applicable facts.
- Route absent-host derivation through the common resolved-context owner and
remove the duplicate derivation from the old validator. Keep the existing
directory/access checks and close-error handling at their appropriate points.
- Preserve deferred discovery/ordinary-overlay checks, root-only absent host
context, host/native distinctions, diagnostics, and borrowed filesystem
ownership. Independently required validation must not disappear merely to
reduce filesystem calls.
- Keep internal documentation aligned with the completed handoff. Describe
aggregate accounting as load-wide and repetition/namespace state as
tier-scoped; the new internal overview/import prose currently calls the latter
per-tier accounting despite its distinct role.
**Verification.** Review the non-import call path to confirm that applicable
resolved facts are carried forward and absent-host derivation has one owner.
Reuse the existing host/native, discovery selected/absent, ordinary/layered
overlay, and passive-resource-context tests. Add a focused behavioral regression
only for a consequential gap exposed by the change; do not assert incidental
canonicalization/stat call counts. Run the development check and applicable
ownership race checks after remediation.

View File

@@ -0,0 +1,253 @@
package engine
import "strconv"
type acquisitionPolicy struct {
importDefaults importDefaults
sequencePolicy sequencePolicy
importLimits importLimits
}
type acquisitionState struct {
accounting *importAccounting
versionState *sourceVersionState
}
type tierAcquisitionState struct {
visited map[resourceIdentity]importRoute
hostToken *namespaceToken
}
type inputAcquisitionRequest struct {
input loadInput
role sourceVersionRole
directiveContext rootDirectiveContext
protectionRule *VersionRule
deferAssembly bool
ordinaryOverlay string
}
type importAcquisitionRequest struct {
input inputAcquisitionRequest
main parsedSource
processing *sourceVersionProcessingResult
}
func acquireInput(request inputAcquisitionRequest, policy acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (assembledInput, []Diagnostic) {
source := request.input.source
destination := request.input.destination
accounting := state.accounting
versionState := state.versionState
visited := tier.visited
hostToken := tier.hostToken
role := request.role
sourceID := "source-" + strconv.Itoa(accounting.sources+1)
selectedSource := source
if source.kind == sourceSearch {
resolved, failedSource, diagnostics := resolveSearchInput(source, sourceID, defaultDiscoveryDirectoryHelpers(), defaultDiscoveryHostProbe())
if len(diagnostics) != 0 {
if failedSource != nil {
failedSource.Destination = destination.clone()
failedSource.placementMode = request.input.placementMode
}
return assembledInput{source: resolved, sources: sourceInfos(failedSource), versionState: versionState, accounting: accounting}, diagnostics
}
selectedSource = resolved
if request.input.filesystem != nil {
if diagnostics := filesystemOptionDiagnostics(selectedSource, request.input.filesystem); len(diagnostics) != 0 {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, diagnostics
}
}
if selectedSource.kind == sourceNone && activeOverlay(request.ordinaryOverlay) && request.input.filesystem == nil {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context")}
}
}
limits := policy.importLimits
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
parsed, diagnostics := parseSource(selectedSource, sourceID, parseLimits)
if parsed.source != nil {
parsed.source.Destination = destination.clone()
parsed.source.placementMode = request.input.placementMode
}
input := assembledInput{source: selectedSource, parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState, accounting: accounting}
if len(diagnostics) != 0 {
input.sources = sourceInfos(parsed.source)
// Mapping value issues already identify application content. Other early
// failures describe source syntax or access, not a placed value.
if selectedSource.kind == sourceMapping && selectedSource.mapIssue != nil && len(destination.segments) != 0 {
for index := range diagnostics {
relative := diagnostics[index].Path
segments := clonePathSegments(destination.segments, len(relative.segments))
diagnostics[index].Path = Path{segments: append(segments, relative.segments...)}
}
}
return input, diagnostics
}
directiveContext := request.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if !request.input.imports && parsed.document != nil {
var directiveDiagnostics []Diagnostic
if directiveContext == rootDirectiveContextSource || directiveContext == rootDirectiveContextLayered {
directiveDiagnostics = disabledRootImportDiagnostics(parsed.document)
} else {
directiveDiagnostics = disabledDirectiveDiagnostics(parsed.document, Path{})
}
if len(directiveDiagnostics) != 0 {
return input, directiveDiagnostics
}
}
var versionProcessing *sourceVersionProcessingResult
if directiveContext == rootDirectiveContextSource {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
var processed sourceVersionProcessingResult
var failures []Diagnostic
if versionState != nil {
processed, failures = versionState.processWithDirectiveContext(parsed.document, role, destination, sourceID, request.input.placementMode, directiveContext)
} else {
processed = processSourceVersionWithDirectiveContext(parsed.document, request.protectionRule, role, destination, sourceVersionModeDefault, directiveContext)
failures = processed.diagnostics
}
if len(failures) != 0 {
return input, failures
}
versionProcessing = &processed
if role == sourceVersionMain && versionState != nil {
versionState.mainProcessing = &processed
}
} else if directiveContext == rootDirectiveContextLayered {
extracted, failures := extractRootDirectives(parsed.document, rootDirectiveContextLayered)
if len(failures) != 0 {
return input, failures
}
processed := sourceVersionProcessingResult{
payload: extracted.body, imports: extracted.imports, sequences: extracted.sequences,
directiveContext: rootDirectiveContextLayered,
}
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
versionProcessed, versionFailures := versionState.process(extracted.body, role, destination, sourceID, request.input.placementMode)
if len(versionFailures) != 0 {
return input, versionFailures
}
versionProcessed.imports = extracted.imports
versionProcessed.sequences = append(cloneSequenceDeclarations(extracted.sequences), versionProcessed.sequences...)
versionProcessed.directiveContext = rootDirectiveContextLayered
processed = versionProcessed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
versionProcessing = &processed
} else if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
processed, failures := versionState.process(parsed.document, role, destination, sourceID, request.input.placementMode)
if len(failures) != 0 {
return input, failures
}
versionProcessing = &processed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
request.input.source = selectedSource
if request.input.imports && selectedSource.kind != sourceNone {
importRequest := importAcquisitionRequest{input: request, main: parsed, processing: versionProcessing}
input.composition, input.sources, input.resolvedFilesystem, diagnostics = acquireImportedInput(importRequest, &policy, state, tier)
if len(diagnostics) != 0 {
return input, diagnostics
}
if input.composition != nil {
input.acquisition = cloneAcquisitionScopes(input.composition.acquisition)
input.document = input.composition.root
}
} else {
input.sources = sourceInfos(parsed.source)
if versionProcessing != nil {
input.document = versionProcessing.payload
}
if request.input.filesystem != nil && (selectedSource.kind == sourceNone || !activeOverlay(request.ordinaryOverlay)) {
if diagnostic := validateFilesystemContext(selectedSource, request.input.filesystem, parsed); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
}
}
if !request.input.imports && input.document != nil {
_, placed, err := placeDocumentWithMode(input.document, destination, request.input.placementMode, input.document.Location, parseLimits.document.Depth, parseLimits.document.Nodes)
if err != nil {
return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
}
localPayload := input.document
input.document = placed
identity := importKeyForMainSource(parsed.source)
var sequenceDeclarations []sequenceDeclarationSpec
if versionProcessing != nil {
sequenceDeclarations = cloneSequenceDeclarations(versionProcessing.sequences)
}
input.acquisition = []acquisitionScope{{
owner: identity,
destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(*parsed.source), payload: localPayload,
identity: identity, role: importRoleMain, scopeOwner: identity,
document: placed, destination: destination.clone(), processing: versionProcessing,
sequenceDeclarations: sequenceDeclarations,
}},
}}
rawNodes := countInputNodes(parsed.document)
placementNodes := destinationWrapperNodeCountForMode(destination, request.input.placementMode)
if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+rawNodes+placementNodes > limits.nodes {
return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the aggregate input limit")}
}
if diagnostic := registerAcquiredSource(selectedSource, request.input.filesystem, parsed.source, visited, hostToken); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
accounting.sources++
accounting.bytes += parsed.bytes
accounting.nodes += rawNodes + placementNodes
}
if !activeOverlay(request.ordinaryOverlay) || request.input.imports {
if input.resolvedFilesystem == nil && len(input.sources) != 0 && (selectedSource.kind == sourceFile || request.input.filesystem != nil || nativeContext(selectedSource, request.input.filesystem)) {
resolved, diagnostic := resolveFilesystemContext(selectedSource, request.input.filesystem, parsed, tier.hostToken)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
input.resolvedFilesystem = &resolved
}
contexts, diagnostic := contextsForResolvedSources(input.resolvedFilesystem, input.sources)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
input.contexts = contexts
}
return input, nil
}
func registerAcquiredSource(source Source, context *FilesystemContext, info *SourceInfo, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) *Diagnostic {
if visited == nil || info == nil || info.Path == "" {
return nil
}
identity := resourceIdentity{namespace: hostToken, path: info.Path}
if source.kind == sourceFilesystem {
identity.namespace = source.namespace.token
} else if context != nil && context.FS != nil {
identity.namespace = context.FS.namespace.token
}
if previous, repeated := visited[identity]; repeated {
value := importRepeatedDiagnostic(Location{SourceID: info.ID}, previous, importRoute{})
return &value
}
visited[identity] = importRoute{canonical: info.Path}
return nil
}

View File

@@ -216,9 +216,8 @@ func TestImportedEmptyBodiesChargeFullRelativeDestinations(t *testing.T) {
currentLimits := limits
currentLimits.nodes = wantNodes + test.delta
accounting := &importAccounting{}
composed, sources, diagnostics := loadImportedDocumentsTracked(
File(mainPath), nil, main, entryDestination, nil, currentLimits, accounting, nil, nil,
)
request := testImportRequest(File(mainPath), nil, main, entryDestination, nil, nil)
composed, sources, diagnostics := testAcquireImportWithAccounting(request, currentLimits, accounting)
if test.wantError {
if composed != nil || len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("below-bound relative placement = %#v, %#v", composed, diagnostics)
@@ -279,9 +278,8 @@ func TestGlobMatchesChargeTheirSharedRelativeDestination(t *testing.T) {
currentLimits := limits
currentLimits.nodes = wantNodes + test.delta
accounting := &importAccounting{}
composed, sources, diagnostics := loadImportedDocumentsTracked(
File(mainPath), nil, main, entryDestination, nil, currentLimits, accounting, nil, nil,
)
request := testImportRequest(File(mainPath), nil, main, entryDestination, nil, nil)
composed, sources, diagnostics := testAcquireImportWithAccounting(request, currentLimits, accounting)
if test.wantError {
if composed != nil || len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("below-bound glob placement = %#v, %#v", composed, diagnostics)
@@ -366,9 +364,8 @@ func TestAbsentOptionalImportsDoNotChargeUnusedDeepDestinations(t *testing.T) {
main := parseImportMain(t, mainPath)
limits := defaultImportLimits()
accounting := &importAccounting{}
composed, sources, diagnostics := loadImportedDocumentsTracked(
File(mainPath), nil, main, destination, nil, limits, accounting, nil, nil,
)
request := testImportRequest(File(mainPath), nil, main, destination, nil, nil)
composed, sources, diagnostics := testAcquireImportWithAccounting(request, limits, accounting)
if len(diagnostics) != 0 || composed == nil || len(sources) != 1 {
t.Fatalf("absent optional import = %#v, %#v, %#v", composed, sources, diagnostics)
}
@@ -412,7 +409,7 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) {
overlay: "test", overlayFilesystem: &FilesystemContext{Root: ".", BaseDir: ".", FS: &filesystem},
}
accounting := &importAccounting{nodes: limits.nodes - baseNodes}
_, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting)
_, diagnostics := testAssembleLayeredInput(request, assembledInput{}, accounting)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverlayLimitExceeded {
t.Fatalf("overlay diagnostics = %#v", diagnostics)
}
@@ -431,7 +428,7 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) {
overrides: []Override{override},
}
accounting := &importAccounting{nodes: limits.nodes - baseNodes - overrideNodes + 1}
input, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting)
input, diagnostics := testAssembleLayeredInput(request, assembledInput{}, accounting)
if len(diagnostics) != 0 {
t.Fatalf("assembly diagnostics = %#v", diagnostics)
}
@@ -443,3 +440,15 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) {
}
})
}
func testAssembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
if accounting == nil {
accounting = &importAccounting{}
}
policy := acquisitionPolicy{
importDefaults: snapshotImportDefaults(request.importDefaults),
sequencePolicy: request.sequencePolicy,
importLimits: defaultImportLimits(),
}
return assembleLayeredInput(request, input, policy, &acquisitionState{accounting: accounting, versionState: input.versionState})
}

View File

@@ -108,8 +108,11 @@ parent: true
}
defer func() { _ = access.close() }()
state := importTraversal{
access: access, accounting: &importAccounting{}, limits: defaultImportLimits(),
active: make(map[resourceIdentity]int), visited: make(map[resourceIdentity]importRoute),
access: access,
policy: &acquisitionPolicy{importLimits: defaultImportLimits()},
state: &acquisitionState{accounting: &importAccounting{}},
tier: &tierAcquisitionState{visited: make(map[resourceIdentity]importRoute)},
active: make(map[resourceIdentity]int),
}
if diagnostics := state.walkDeclarations(directory, declarations, 0, Keys("entry")); len(diagnostics) != 0 {
t.Fatalf("walk diagnostics = %#v", diagnostics)
@@ -160,11 +163,10 @@ func TestEntryDestinationSurvivesImportSetupAndParseFailures(t *testing.T) {
t.Run("import context failure", func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: child.yml}]}\n")
input, diagnostics := assembleBaseInputTracked(
File(mainPath),
Options[map[string]any]{Imports: true, Filesystem: &FilesystemContext{Root: t.TempDir()}},
Keys("mounted"), &importAccounting{}, nil, nil,
)
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true, filesystem: &FilesystemContext{Root: t.TempDir()}, destination: Keys("mounted")},
role: sourceVersionMain, directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, nil)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportOutsideRoot {
t.Fatalf("context diagnostics = %#v", diagnostics)
}
@@ -177,10 +179,10 @@ func TestEntryDestinationSurvivesImportSetupAndParseFailures(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: broken.yml}]}\n")
writeImportFile(t, directory, "broken.yml", "broken: [\n")
input, diagnostics := assembleBaseInputTracked(
File(mainPath), Options[map[string]any]{Imports: true},
Keys("mounted"), &importAccounting{}, nil, nil,
)
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true, destination: Keys("mounted")},
role: sourceVersionMain, directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, nil)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeYAMLInvalid {
t.Fatalf("parse diagnostics = %#v", diagnostics)
}

View File

@@ -3,7 +3,6 @@ package engine
import (
"io/fs"
"os"
"path"
"path/filepath"
"strings"
)
@@ -13,17 +12,20 @@ func validateFilesystemContext(source Source, context *FilesystemContext, main p
return validateAbsentHostFilesystemContext(context)
}
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
resolved, diagnostic := resolveNativeFilesystemContext(source, context)
if diagnostic != nil {
return diagnostic
}
return validateNativeDirectories(access, basePath, mainPath)
return validateResolvedFilesystemDirectories(resolved)
}
rootPath, _, _, diagnostic := importFilesystem(source, context, main)
resolved, diagnostic := resolveHostFilesystemContext(source, context, main, nil)
if diagnostic != nil {
return diagnostic
}
root, err := os.OpenRoot(rootPath)
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
return diagnostic
}
root, err := os.OpenRoot(resolved.rootPath)
if err != nil {
value := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
value.Primary = nil
@@ -134,113 +136,6 @@ func nativeContext(source Source, context *FilesystemContext) bool {
return source.kind == sourceFilesystem || context != nil && context.FS != nil
}
// nativeFilesystemAccess resolves namespace context without opening caller
// resources. File inputs always derive their base from the selected file.
func nativeFilesystemAccess(source Source, context *FilesystemContext) (*filesystemAccess, string, string, *Diagnostic) {
var namespace *nativeNamespace
rootPath := "."
basePath := "."
mainPath := ""
if source.kind == sourceFilesystem {
namespace = source.namespace
mainPath = source.path
basePath = path.Dir(source.path)
if context != nil && context.Root != "" {
rootPath = context.Root
}
} else if context != nil && context.FS != nil {
namespace = context.FS.namespace
rootPath, basePath = context.Root, context.BaseDir
}
if namespace == nil || nilFilesystem(namespace.filesystem) || !fs.ValidPath(rootPath) || !fs.ValidPath(basePath) {
value := optionDiagnostic(CodeOptionsInvalid, "filesystem context is invalid")
return nil, "", "", &value
}
access := newNativeAccess(namespace.filesystem, rootPath, namespace.token)
access.label = namespace.label
if !access.contains(basePath) || mainPath != "" && !access.contains(mainPath) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return nil, "", "", &value
}
return access, basePath, mainPath, nil
}
func validateNativeDirectories(access *filesystemAccess, basePath, mainPath string) *Diagnostic {
if !access.contains(basePath) || mainPath != "" && !access.contains(mainPath) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return &value
}
for _, directory := range []string{access.rootPath, basePath} {
info, err := access.stat(directory)
if err != nil || !info.IsDir() {
value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)
return &value
}
}
return nil
}
func importFilesystem(source Source, context *FilesystemContext, main parsedSource) (rootPath, basePath, mainPath string, diagnostic *Diagnostic) {
if main.source == nil || main.document == nil {
value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil)
return "", "", "", &value
}
if source.kind == sourceFile {
if context != nil && context.BaseDir != "" {
value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")
return "", "", "", &value
}
canonical, err := canonicalExistingPath(source.path)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil)
return "", "", "", &value
}
mainPath = canonical
basePath = filepath.Dir(canonical)
rootPath = basePath
if context != nil && context.Root != "" {
rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return "", "", "", &value
}
}
} else {
if context == nil || context.BaseDir == "" || context.Root == "" {
value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories")
return "", "", "", &value
}
if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) {
value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute")
return "", "", "", &value
}
var err error
basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return "", "", "", &value
}
rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return "", "", "", &value
}
}
if !containedPath(rootPath, basePath) || (mainPath != "" && !containedPath(rootPath, mainPath)) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return "", "", "", &value
}
for _, candidate := range []string{rootPath, basePath} {
info, err := os.Stat(candidate)
if err != nil || !info.IsDir() {
value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)
return "", "", "", &value
}
}
return rootPath, basePath, mainPath, nil
}
func canonicalExistingPath(name string) (string, error) {
absolute, err := filepath.Abs(name)
if err != nil {

View File

@@ -0,0 +1,176 @@
package engine
import (
"io/fs"
"os"
"path"
"path/filepath"
"strings"
)
type resolvedFilesystemContext struct {
native *nativeNamespace
hostToken *namespaceToken
rootPath string
basePath string
mainPath string
}
func resolveFilesystemContext(source Source, context *FilesystemContext, main parsedSource, hostToken *namespaceToken) (resolvedFilesystemContext, *Diagnostic) {
if source.kind == sourceNone && context != nil {
return resolveAbsentFilesystemContext(context)
}
if nativeContext(source, context) {
return resolveNativeFilesystemContext(source, context)
}
return resolveHostFilesystemContext(source, context, main, hostToken)
}
func resolveAbsentFilesystemContext(context *FilesystemContext) (resolvedFilesystemContext, *Diagnostic) {
if context.FS != nil {
return resolveNativeFilesystemContext(Source{kind: sourceNone}, context)
}
rootPath, err := canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
basePath := rootPath
if context.BaseDir != "" {
basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
}
if !containedPath(rootPath, basePath) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return resolvedFilesystemContext{}, &value
}
return resolvedFilesystemContext{rootPath: rootPath, basePath: basePath}, nil
}
func resolveNativeFilesystemContext(source Source, context *FilesystemContext) (resolvedFilesystemContext, *Diagnostic) {
var namespace *nativeNamespace
resolved := resolvedFilesystemContext{rootPath: ".", basePath: "."}
if source.kind == sourceFilesystem {
namespace = source.namespace
resolved.mainPath = source.path
resolved.basePath = path.Dir(source.path)
if context != nil && context.Root != "" {
resolved.rootPath = context.Root
}
} else if context != nil && context.FS != nil {
namespace = context.FS.namespace
resolved.rootPath, resolved.basePath = context.Root, context.BaseDir
}
if namespace == nil || nilFilesystem(namespace.filesystem) || !fs.ValidPath(resolved.rootPath) || !fs.ValidPath(resolved.basePath) {
value := optionDiagnostic(CodeOptionsInvalid, "filesystem context is invalid")
return resolvedFilesystemContext{}, &value
}
resolved.native = namespace
if !containsNativePath(resolved.rootPath, resolved.basePath) || resolved.mainPath != "" && !containsNativePath(resolved.rootPath, resolved.mainPath) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return resolvedFilesystemContext{}, &value
}
return resolved, nil
}
func resolveHostFilesystemContext(source Source, context *FilesystemContext, main parsedSource, hostToken *namespaceToken) (resolvedFilesystemContext, *Diagnostic) {
if main.source == nil {
value := importDiagnostic(CodeImportInvalid, "imports require a selected configuration source", nil)
return resolvedFilesystemContext{}, &value
}
resolved := resolvedFilesystemContext{hostToken: hostToken}
if source.kind == sourceFile {
if context != nil && context.BaseDir != "" {
value := optionDiagnostic(CodeOptionsInvalid, "file sources determine their own import base directory")
return resolvedFilesystemContext{}, &value
}
canonical := main.source.Path
if canonical == "" {
var err error
canonical, err = canonicalExistingPath(source.path)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "selected source path could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
}
resolved.mainPath = canonical
resolved.basePath = filepath.Dir(canonical)
resolved.rootPath = resolved.basePath
if context != nil && context.Root != "" {
rootPath, err := canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
resolved.rootPath = rootPath
}
} else {
if context == nil || context.BaseDir == "" || context.Root == "" {
value := optionDiagnostic(CodeOptionsInvalid, "byte and reader imports require absolute base and root directories")
return resolvedFilesystemContext{}, &value
}
if !filepath.IsAbs(context.BaseDir) || !filepath.IsAbs(context.Root) {
value := optionDiagnostic(CodeOptionsInvalid, "byte and reader import base and root directories must be absolute")
return resolvedFilesystemContext{}, &value
}
var err error
resolved.basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
resolved.rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return resolvedFilesystemContext{}, &value
}
}
if !containedPath(resolved.rootPath, resolved.basePath) || resolved.mainPath != "" && !containedPath(resolved.rootPath, resolved.mainPath) {
value := importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)
return resolvedFilesystemContext{}, &value
}
return resolved, nil
}
func containsNativePath(root, name string) bool {
if !fs.ValidPath(name) || !fs.ValidPath(root) {
return false
}
return root == "." || name == root || strings.HasPrefix(name, root+"/")
}
func validateResolvedFilesystemDirectories(resolved resolvedFilesystemContext) *Diagnostic {
for _, directory := range []string{resolved.rootPath, resolved.basePath} {
var info fs.FileInfo
var err error
if resolved.native != nil {
info, err = fs.Stat(resolved.native.filesystem, directory)
} else {
info, err = os.Stat(directory)
}
if err != nil || !info.IsDir() {
value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)
return &value
}
}
return nil
}
func openResolvedFilesystemAccess(resolved resolvedFilesystemContext) (*filesystemAccess, error) {
if resolved.native != nil {
access := newNativeAccess(resolved.native.filesystem, resolved.rootPath, resolved.native.token)
access.label = resolved.native.label
return access, nil
}
access, err := openHostAccess(resolved.rootPath)
if err != nil {
return nil, err
}
if resolved.hostToken != nil {
access.token = resolved.hostToken
}
return access, nil
}

View File

@@ -216,7 +216,8 @@ func TestGlobTraversalBoundsNonmatchesAndAcceptedCandidates(t *testing.T) {
main := parseImportMain(t, mainPath)
limits := defaultImportLimits()
limits.entries = 2
_, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, limits)
request := testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
_, _, diagnostics := testAcquireImportWithLimits(request, limits)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("entry-budget diagnostics = %#v", diagnostics)
}
@@ -228,7 +229,8 @@ func TestGlobTraversalBoundsNonmatchesAndAcceptedCandidates(t *testing.T) {
main = parseImportMain(t, mainPath)
limits = defaultImportLimits()
limits.sources = 2
_, _, diagnostics = loadImportedDocuments(File(mainPath), nil, main, nil, limits)
request = testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
_, _, diagnostics = testAcquireImportWithLimits(request, limits)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("candidate-budget diagnostics = %#v", diagnostics)
}

View File

@@ -0,0 +1,533 @@
package engine
import (
"bytes"
"errors"
"io/fs"
"strconv"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml"
)
type importAdmissionOutcome uint8
const (
importAdmissionReady importAdmissionOutcome = iota
importAdmissionOptionalMissing
)
type admittedImport struct {
canonical string
identity resourceIdentity
nodeKey importNodeKey
source SourceInfo
payload *document.Node
declarations []importDeclarationSpec
sequenceDeclarations []sequenceDeclarationSpec
processing *sourceVersionProcessingResult
}
type importAdmissionResult struct {
outcome importAdmissionOutcome
source admittedImport
}
func acquireImportedInput(request importAcquisitionRequest, policy *acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (*composedDocument, []SourceInfo, *resolvedFilesystemContext, []Diagnostic) {
source := request.input.input.source
context := request.input.input.filesystem
main := request.main
resolved, diagnostic := resolveFilesystemContext(source, context, main, tier.hostToken)
if diagnostic != nil {
return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic}
}
if resolved.native == nil {
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
return nil, sourceInfos(main.source), nil, []Diagnostic{*diagnostic}
}
}
access, err := openResolvedFilesystemAccess(resolved)
if err != nil {
diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
diagnostic.Primary = nil
return nil, sourceInfos(main.source), nil, []Diagnostic{diagnostic}
}
composition, sources, diagnostics := acquireImportedWithAccess(request, policy, state, tier, access, resolved)
if len(diagnostics) != 0 {
return composition, sources, nil, diagnostics
}
return composition, sources, &resolved, nil
}
func acquireImportedWithAccess(request importAcquisitionRequest, policy *acquisitionPolicy, loadState *acquisitionState, tier *tierAcquisitionState, access *filesystemAccess, resolved resolvedFilesystemContext) (*composedDocument, []SourceInfo, []Diagnostic) {
main := request.main
destination := request.input.input.destination
limits := policy.importLimits
accounting := loadState.accounting
versionState := loadState.versionState
processing := request.processing
basePath, mainPath := resolved.basePath, resolved.mainPath
if tier.visited == nil {
tier.visited = make(map[resourceIdentity]importRoute)
}
visited := tier.visited
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
}
for _, directory := range []string{access.rootPath, basePath} {
info, err := access.stat(directory)
if err != nil || !info.IsDir() {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)}
}
}
mainInfo := cloneSourceInfo(*main.source)
mainInfo.Destination = destination.clone()
mainIsItem := main.source.placementMode == placementModeSequenceItem
if mainPath != "" {
mainInfo.Path = mainPath
}
mainResource := access.identity(mainPath)
mainNodeKey := importKeyFor(access, mainPath, mainPath == "")
if mainPath == "" {
mainResource = access.syntheticIdentity()
}
if previous, repeated := visited[mainResource]; repeated {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})}
}
accounting.sources++
accounting.bytes += main.bytes
accounting.nodes += countInputNodes(main.document)
directiveContext := request.input.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if processing != nil && processing.directiveContext != 0 {
directiveContext = processing.directiveContext
}
request.input.directiveContext = directiveContext
state := importTraversal{
access: access, request: request, policy: policy, state: loadState, tier: tier,
scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()},
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int),
importGraph: newImportInclusionGraph(),
}
if mainIsItem {
state.itemScope = true
state.itemPrefix = destination.clone()
state.itemFallback = destination.clone()
state.scope.destination = Path{}
state.sources[0] = cloneSourceInfo(mainInfo)
}
if diagnostic := state.checkInitialLimits(); diagnostic != nil {
_ = access.close()
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
}
mainIdentity := mainPath
if mainIdentity == "" {
mainIdentity = "<selected-source>"
}
mainRoute := importRoute{canonical: mainIdentity}
state.importGraph.addNode(importGraphNode{key: mainNodeKey, canonical: mainIdentity, sourceID: mainInfo.ID, role: importRoleMain})
mainRoute.key = mainNodeKey
state.active[mainResource] = 0
tier.visited[mainResource] = mainRoute
state.stack = append(state.stack, mainRoute)
var body *document.Node
var declarations []importDeclarationSpec
var sequenceDeclarations []sequenceDeclarationSpec
var diagnostics []Diagnostic
if processing != nil {
body = processing.payload
declarations = processing.imports
sequenceDeclarations = cloneSequenceDeclarations(processing.sequences)
} else {
body, declarations, diagnostics = splitImportDirectives(main.document)
}
if len(diagnostics) == 0 {
if mainIsItem {
state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain,
scopeOwner: mainNodeKey, payload: body, document: body, processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{})
} else {
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
placementNodes := destinationWrapperNodeCount(destination)
if err != nil {
diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
} else if accounting.nodes+placementNodes > limits.nodes {
diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
} else {
accounting.nodes += placementNodes
state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), payload: body, document: placed,
identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey,
destination: destination.clone(), processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
diagnostics = state.walkDeclarations(basePath, declarations, 0, destination)
}
}
}
if len(diagnostics) == 0 {
state.scheduleAcquisitionScopes(&state.scope)
}
closeErr := access.close()
if len(diagnostics) != 0 {
if mainIsItem {
diagnostics = prependCompositionDiagnosticPaths(diagnostics, destination.Index(0))
}
return nil, cloneSourceInfos(state.sources), diagnostics
}
if closeErr != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
}
acquisition := state.scope
if mainIsItem {
localScope := state.scope
acquisition = acquisitionScope{
owner: mainNodeKey, destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), pendingItem: &localScope,
}},
}
}
if request.input.deferAssembly {
if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
}
return &composedDocument{
acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}, cloneSourceInfos(state.sources), nil
}
acquired := &composedDocument{
acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}
input := assembledInput{
parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition),
sources: state.sources, versionState: versionState, accounting: accounting,
}
composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy.sequencePolicy)
return composed, cloneSourceInfos(state.sources), diagnostics
}
func (s *importTraversal) checkInitialLimits() *Diagnostic {
if s.state.accounting.sources > s.policy.importLimits.sources {
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil)
return &value
}
if s.state.accounting.bytes > s.policy.importLimits.bytes {
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil)
return &value
}
if s.state.accounting.nodes > s.policy.importLimits.nodes {
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)
return &value
}
return nil
}
func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic {
policies := make([]resolvedImportPolicy, len(declarations))
for index, spec := range declarations {
policy, diagnostics := resolveImportPolicy(spec, s.policy.importDefaults)
if len(diagnostics) != 0 {
return diagnostics
}
if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 {
site := spec.fieldSites["placement"]
return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}}
}
policies[index] = policy
}
for index, spec := range declarations {
policy := policies[index]
var selection importSelection
var diagnostics []Diagnostic
switch spec.selector {
case importSelectorPath:
selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy)
case importSelectorDirectory:
selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting)
case importSelectorGlob:
selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.policy.importLimits, s.state.accounting)
default:
location := spec.location
return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)}
}
if len(diagnostics) != 0 {
return diagnostics
}
childDestination := appendDestination(parentDestination, policy.destination)
// Count admitted records rather than lexical candidates: optional missing
// paths are omitted during admission, and a partially present list is not empty.
sourcesBefore := len(s.sources)
for _, candidate := range selection.candidates {
if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 {
return diagnostics
}
}
if !policy.required && len(s.sources) == sourcesBefore {
s.recordEmptyImportSelection(spec)
}
}
return nil
}
func (s *importTraversal) recordEmptyImportSelection(spec importDeclarationSpec) {
for index := range s.sources {
if s.sources[index].ID != spec.location.SourceID {
continue
}
s.sources[index].importSelections = append(s.sources[index].importSelections, importSelectionEvidence{
path: spec.path.clone(), location: spec.location,
selector: spec.selector, outcome: importSelectionNone,
})
return
}
}
func (s *importTraversal) admitImportSource(canonical string, identity resourceIdentity, nodeKey importNodeKey, candidate selectedImportCandidate, policy resolvedImportPolicy, destination Path, location Location) (importAdmissionResult, []Diagnostic) {
sourceID := "source-" + strconv.Itoa(s.state.accounting.sources+1)
kind := SourceKindFile
label := candidate.label
if label == "" {
label = candidate.path
}
if s.access.native != nil {
kind = SourceKindFilesystem
label = s.access.label + "/" + canonical
}
info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location}
if policy.placement == placementModeSequenceItem {
info.placementMode = placementModeSequenceItem
}
// A source's own item placement still has an exact sequence-container
// destination. Only an unresolved enclosing item index makes the path a
// fallback container for this snapshot.
info.destinationUnresolved = s.itemScope
if kind == SourceKindFilesystem {
info.namespace = s.access.token
}
s.sources = append(s.sources, info)
data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required)
if diagnostic != nil {
return importAdmissionResult{}, []Diagnostic{*diagnostic}
}
if skipped {
s.sources = s.sources[:len(s.sources)-1]
return importAdmissionResult{outcome: importAdmissionOptionalMissing}, nil
}
s.state.accounting.sources++
root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{
Depth: maxDocumentDepth, Nodes: maxDocumentNodes,
})
if len(issues) != 0 {
return importAdmissionResult{}, parserDiagnostics(sourceID, issues)
}
nodeCount := countInputNodes(root)
if s.state.accounting.nodes+nodeCount > s.policy.importLimits.nodes {
value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)
return importAdmissionResult{}, []Diagnostic{value}
}
s.state.accounting.nodes += nodeCount
directiveContext := s.request.input.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
versionDestination := destination
if s.itemScope {
versionDestination = appendDestination(s.itemPrefix.Index(0), destination)
}
var processing sourceVersionProcessingResult
if s.state.versionState != nil && s.state.versionState.mode != sourceVersionModeDefault {
processed, failures := s.state.versionState.processWithDirectiveContext(root, sourceVersionContribution, versionDestination, sourceID, policy.placement, directiveContext)
if len(failures) != 0 {
return importAdmissionResult{}, failures
}
processing = processed
} else {
processed := processSourceVersionWithDirectiveContext(root, s.request.input.protectionRule, sourceVersionContribution, versionDestination, sourceVersionModeDefault, directiveContext)
if len(processed.diagnostics) != 0 {
return importAdmissionResult{}, processed.diagnostics
}
processing = processed
}
return importAdmissionResult{source: admittedImport{
canonical: canonical, identity: identity, nodeKey: nodeKey,
source: cloneSourceInfo(info), payload: processing.payload,
declarations: processing.imports,
sequenceDeclarations: cloneSequenceDeclarations(processing.sequences),
processing: &processing,
}}, nil
}
func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic {
location := candidate.site.location
if location.SourceID == "" && location.Line == 0 && location.Column == 0 {
location = spec.location
}
if !s.access.contains(candidate.accessPath) {
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)}
}
canonical, err := s.access.canonical(candidate.accessPath)
if err != nil {
if !policy.required && errors.Is(err, fs.ErrNotExist) {
return nil
}
code := CodeImportInvalid
message := "import target could not be canonicalized"
if errors.Is(err, fs.ErrNotExist) {
code, message = CodeImportMissing, "required import target does not exist"
} else if errors.Is(err, fs.ErrPermission) {
message = "import target cannot be accessed"
}
return []Diagnostic{importDiagnostic(code, message, &location)}
}
if !s.access.contains(canonical) {
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)}
}
if depth > s.policy.importLimits.depth {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
}
currentRoute := s.currentRoute(location)
identity := s.access.identity(canonical)
nodeKey := importKeyFor(s.access, canonical, false)
importerKey := s.currentImportNode()
if activeIndex, ok := s.active[identity]; ok {
closingEdge := s.importGraph.addEdge(importerKey, nodeKey, location)
if diagnostic, ok := s.importGraph.scheduleCycle(closingEdge, location); ok {
return []Diagnostic{diagnostic}
}
cycle := append([]importRoute(nil), s.stack[activeIndex:]...)
return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)}
}
if previous, ok := s.tier.visited[identity]; ok {
s.importGraph.addEdge(importerKey, nodeKey, location)
return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)}
}
if s.state.accounting.sources+1 > s.policy.importLimits.sources {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)}
}
if _, err := s.access.relative(canonical); err != nil {
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)}
}
admission, diagnostics := s.admitImportSource(canonical, identity, nodeKey, candidate, policy, destination, location)
if len(diagnostics) != 0 {
return diagnostics
}
if admission.outcome == importAdmissionOptionalMissing {
return nil
}
admitted := admission.source
placementDestination := destination
mode := placementModeMapping
if policy.placement == placementModeSequenceItem {
mode = placementModeSequenceItem
}
role := importRoleMapping
if mode == placementModeSequenceItem {
role = importRoleItem
}
s.importGraph.addNode(importGraphNode{key: admitted.nodeKey, canonical: admitted.canonical, sourceID: admitted.source.ID, role: role})
s.importGraph.addEdge(importerKey, admitted.nodeKey, location)
if mode == placementModeSequenceItem {
route := importRoute{canonical: admitted.canonical, locations: currentRoute.locations, key: admitted.nodeKey}
s.active[admitted.identity] = len(s.stack)
s.tier.visited[admitted.identity] = route
s.stack = append(s.stack, route)
diagnostics := s.visitItemSource(admitted, destination, relativeDestination, depth)
delete(s.active, admitted.identity)
s.stack = s.stack[:len(s.stack)-1]
return diagnostics
}
_, placed, err := placeDocument(admitted.payload, placementDestination, admitted.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)}
}
placementNodes := destinationWrapperNodeCount(placementDestination)
if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
}
s.state.accounting.nodes += placementNodes
route := importRoute{canonical: admitted.canonical, locations: currentRoute.locations, key: admitted.nodeKey}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(admitted.source), payload: admitted.payload, document: placed,
identity: admitted.nodeKey, role: role, scopeOwner: s.scope.owner,
destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: admitted.processing,
sequenceDeclarations: cloneSequenceDeclarations(admitted.sequenceDeclarations),
})
s.active[admitted.identity] = len(s.stack)
s.tier.visited[admitted.identity] = route
s.stack = append(s.stack, route)
diagnostics = s.walkDeclarations(s.access.dir(admitted.canonical), admitted.declarations, depth, destination)
delete(s.active, admitted.identity)
s.stack = s.stack[:len(s.stack)-1]
return diagnostics
}
func (s *importTraversal) currentImportNode() importNodeKey {
if len(s.stack) != 0 {
return s.stack[len(s.stack)-1].key
}
key := s.scope.owner
if _, exists := s.importGraph.index[key]; !exists {
s.importGraph.addNode(importGraphNode{key: key, canonical: "<selected-source>", role: importRoleMain})
}
return key
}
func (s *importTraversal) sourceDestination(destination Path) Path {
if !s.itemScope {
return destination.clone()
}
// Until the enclosing item receives its assigned index, only the collection
// container is exact. Keep the local suffix in the acquisition projection.
return s.itemFallback.clone()
}
func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) {
remaining := s.policy.importLimits.bytes - s.state.accounting.bytes
limit := min(int64(maxSourceBytes), remaining)
if limit < 0 {
limit = 0
}
data, failure := readRegularFile(name, s.access.stat, s.access.open, limit)
if failure != nil {
if !required && failure.missingTarget {
return nil, true, nil
}
value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit")
return nil, false, &value
}
s.state.accounting.bytes += int64(len(data))
return data, false, nil
}
func (s *importTraversal) currentRoute(location Location) importRoute {
locations := make([]Location, 0, len(s.stack)+1)
for _, route := range s.stack {
if len(route.locations) != 0 {
locations = append(locations, route.locations[len(route.locations)-1])
}
}
locations = append(locations, location)
return importRoute{locations: locations}
}

View File

@@ -0,0 +1,277 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func prependCompositionDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic {
cloned := append([]Diagnostic(nil), diagnostics...)
for index := range cloned {
if !isCompositionDiagnostic(cloned[index]) {
continue
}
cloned[index].Path = appendDestination(prefix, cloned[index].Path)
cloned[index].Primary = cloneLocation(cloned[index].Primary)
cloned[index].Related = cloneLocations(cloned[index].Related)
}
return cloned
}
func isCompositionDiagnostic(diagnostic Diagnostic) bool {
return diagnostic.Code == CodeComposeConflict || diagnostic.Code == CodeComposeInvalid
}
// On a composition failure only the earlier contributions are needed to locate
// this item. Project one enclosing scope at a time as traversal unwinds; syntax
// and access diagnostics keep their original source-local coordinates.
func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path, beforeContribution int) []Diagnostic {
for _, diagnostic := range diagnostics {
if isCompositionDiagnostic(diagnostic) {
prefix := parent
prefix.contributions = parent.contributions[:beforeContribution]
index := priorSequenceLength([]acquisitionScope{prefix}, destination)
return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index))
}
}
return diagnostics
}
// This scan is confined to failed loads, not the successful construction path.
func priorSequenceLength(scopes []acquisitionScope, path Path) int {
length := 0
for _, scope := range scopes {
for _, contribution := range scope.contributions {
if node, present := documentAtPath(contribution.document, path); present && node.Kind == document.Sequence {
length += len(node.Sequence)
}
}
}
return length
}
func projectPriorInputDiagnosticPaths(diagnostics []Diagnostic, scopes []acquisitionScope) []Diagnostic {
cloned := cloneDiagnostics(diagnostics)
for index := range cloned {
if !isCompositionDiagnostic(cloned[index]) {
continue
}
for position, segment := range cloned[index].Path.segments {
if segment.kind != pathSegmentIndex {
continue
}
container := Path{segments: cloned[index].Path.segments[:position]}
cloned[index].Path.segments[position].index += priorSequenceLength(scopes, container)
// Registration does not address paths through item indexes. A prior
// input shifts the outer collection, not its item-local sequences.
break
}
}
return cloned
}
func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
if composed == nil {
return
}
applications := composed.applications
projectedApplications := make([]historyApplication, 0, len(applications))
ranges := indexSequenceRanges(composed.sequenceRanges)
for sourceIndex, contribution := range scope.contributions {
if contribution.nested == nil {
projectedApplications = append(projectedApplications, applications[sourceIndex])
continue
}
itemIndex := 0
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
prefix := contribution.destination.Index(itemIndex)
if sourceIndex < len(applications) {
outer := applications[sourceIndex]
structural := document.Clone(outer.value)
if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > 0 {
// The retained fragment is still a local singleton. Its assigned
// outer index belongs to the projection, not this source tree.
node.Sequence[0] = &document.Node{Kind: document.Mapping, Location: node.Sequence[0].Location}
}
outer.value = structural
projectedApplications = append(projectedApplications, outer)
for _, nested := range contribution.nested.applications {
projected := nested
projected.path = appendDestination(prefix, nested.path)
projected.sequences = cloneSequenceHistoryContributions(nested.sequences)
projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences)
projectedApplications = append(projectedApplications, projected)
}
}
projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix)
}
composed.applications = projectedApplications
projectNestedCollectionPolicies(composed, scope, ranges)
}
func projectNestedCollectionPolicies(composed *composedDocument, scope acquisitionScope, ranges map[sequenceRangeKey]sequenceRange) {
if composed == nil {
return
}
type itemPolicyProjection struct {
destination Path
policy collectionPolicy
base int
}
projected := make(map[string]*itemPolicyProjection)
var prefixLengths map[string]int
for sourceIndex, contribution := range scope.contributions {
if prefixLengths != nil && sourceIndex > 0 {
addSequencePrefixLengths(scope.contributions[sourceIndex-1].document, Path{}, prefixLengths)
}
if contribution.nested == nil || len(contribution.nested.collectionPolicies.collections) == 0 {
continue
}
key := contribution.destination.String()
assigned := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: key}]
itemIndex, found := assigned.outputForInput(0)
if !found {
// Normal composition supplies the assignment directly. For a
// handoff without ranges, scan each preceding tree only once.
if prefixLengths == nil {
prefixLengths = make(map[string]int)
for _, previous := range scope.contributions[:sourceIndex] {
addSequencePrefixLengths(previous.document, Path{}, prefixLengths)
}
}
itemIndex = prefixLengths[key]
}
working, initialized := projected[key]
if !initialized {
// Read one owned copy per destination, then add each incoming
// record without recopying earlier records and their scopes.
policy, _ := composed.collectionPolicies.at(contribution.destination)
working = &itemPolicyProjection{destination: contribution.destination, policy: policy, base: len(policy.recordOrder)}
projected[key] = working
}
working.policy.setRecordPolicyAt(working.base+itemIndex, collectionRecordPolicy{
owner: newCollectionRecordPolicy().owner,
scope: contribution.nested.collectionPolicies,
})
}
for _, working := range projected {
composed.collectionPolicies.set(working.destination, working.policy)
}
}
func addSequencePrefixLengths(node *document.Node, path Path, lengths map[string]int) {
if node == nil {
return
}
if node.Kind == document.Sequence {
lengths[path.String()] += len(node.Sequence)
return
}
// Contribution destinations select mapping paths, not sequence members.
for _, entry := range node.Mapping {
addSequencePrefixLengths(entry.Value, path.Key(entry.Key), lengths)
}
}
func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) {
var visit func(*document.Node, Path)
visit = func(node *document.Node, path Path) {
if node == nil {
return
}
if origin, ok := origins[path.String()]; ok {
target[appendDestination(prefix, path).String()] = cloneOrigin(origin)
}
for _, entry := range node.Mapping {
visit(entry.Value, path.Key(entry.Key))
}
for index, child := range node.Sequence {
visit(child, path.Index(index))
}
}
visit(root, Path{})
}
func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) {
byID := make(map[string]int, len(sources))
for index := range sources {
byID[sources[index].ID] = index
}
var projectScope func(*composedDocument, acquisitionScope, Path, bool, string)
projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) {
ranges := indexSequenceRanges(currentComposition.sequenceRanges)
for sourceIndex, contribution := range current.contributions {
if contribution.source.ID == parentSourceID {
continue
}
contributionPrefix := prefix
contributionInsideItem := insideItem
itemIndex := 0
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
if contribution.source.placementMode == placementModeSequenceItem {
container := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = container.clone()
contribution.source.destinationUnresolved = false
current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source)
if position, ok := byID[contribution.source.ID]; ok {
sources[position].Destination = container
sources[position].destinationUnresolved = false
}
contributionPrefix = appendDestination(container, Path{}.Index(itemIndex))
contributionInsideItem = true
projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex)
} else if contributionInsideItem {
exactDestination := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = exactDestination.clone()
contribution.source.destinationUnresolved = false
current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source)
if position, ok := byID[contribution.source.ID]; ok {
sources[position].Destination = exactDestination
sources[position].destinationUnresolved = false
}
projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex)
} else {
projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex)
}
if contribution.nested != nil && contribution.nested.acquisition != nil {
for _, nestedScope := range contribution.nested.acquisition {
projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID)
}
}
}
}
projectScope(composed, scope, Path{}, false, "")
}
func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) {
if processing != nil && processing.evidence != nil {
evidence := processing.evidence
evidence.destination = destination.clone()
// Rebuild from source-local paths on each handoff: an input can be
// composed first on its own and then at a different offset in its tier.
evidence.projectedPaths = nil
paths := evidence.sensitivityPaths()
for index, path := range paths {
for position := range path.segments {
prefix := Path{segments: path.segments[:position]}
assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}]
if !found {
continue
}
item, ok := schemaIndex(path.segments[position])
if ok && path.segments[position].kind != pathSegmentKey {
if outputIndex, projected := assigned.outputForInput(item); projected {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex}
}
}
}
paths[index] = path
}
evidence.projectedPaths = paths
}
}

View File

@@ -0,0 +1,287 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func compositionTraversal(policy sequencePolicy, rule *VersionRule, versionState *sourceVersionState, accounting *importAccounting, graph importInclusionGraph) importTraversal {
return importTraversal{
request: importAcquisitionRequest{input: inputAcquisitionRequest{protectionRule: rule}},
policy: &acquisitionPolicy{sequencePolicy: policy, importLimits: defaultImportLimits()},
state: &acquisitionState{accounting: accounting, versionState: versionState},
tier: &tierAcquisitionState{},
importGraph: graph.clone(),
}
}
// composeAcquiredInputWithPolicy is the per-input adapter from retained
// acquisition scopes to the established strict-composition behavior. It uses
// the already acquired documents and graph; source handles are closed before
// this handoff.
func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
return input.composition, nil
}
acquired := input.composition
if len(acquired.acquisition) != 1 {
return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)}
}
scope := cloneAcquisitionScope(acquired.acquisition[0])
var rule *VersionRule
if input.versionState != nil {
rule = input.versionState.effectiveRule
}
accounting := input.accounting
if accounting == nil {
accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes}
}
traversal := compositionTraversal(policy, rule, input.versionState, accounting, acquired.importGraph)
if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil {
main := scope.contributions[0]
localScope := cloneAcquisitionScope(*main.pendingItem)
if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
}
documents := make([]*document.Node, 0, len(localScope.contributions))
for _, contribution := range localScope.contributions {
documents = append(documents, contribution.document)
}
completed, diagnostics := composeDocumentsWithCollectionPolicy(
policy, localScope.collectionPolicies,
acquiredSequenceDeclarations(localScope), documents...,
)
if len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
}
if input.versionState != nil {
protectionRule := input.versionState.localRule
if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil {
prefix := main.destination.Index(0)
relative, inside := relativeHistoryPath(prefix, protectionRule.Path)
if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) {
protectionRule = nil
} else {
scoped := *protectionRule
scoped.Path = relative
protectionRule = &scoped
}
}
if diagnostic := importedVersionClaim(documents, localScope, completed, protectionRule); diagnostic != nil {
if rule != nil {
diagnostic.Path = rule.Path.clone()
}
return nil, []Diagnostic{*diagnostic}
}
}
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)}
projectNestedEvidence(completed, localScope)
_, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
}
if !main.placementCharged {
placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > traversal.policy.importLimits.nodes {
return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)}
}
accounting.nodes += placementNodes
}
composition, diagnostics := composeDocumentsWithPolicy(policy, placed)
if len(diagnostics) != 0 {
return nil, diagnostics
}
synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{
source: cloneSourceInfo(main.source), payload: completed.root, document: placed,
identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner,
destination: main.destination.clone(), processing: main.processing, nested: completed,
}}}
projectNestedEvidence(composition, synthetic)
projectAcquiredDestinations(composition, synthetic, input.sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = acquired.importGraph.clone()
return composition, nil
}
return composeAcquiredScopeWithPolicy(
&scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting,
)
}
// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group
// handoffs. Lower policy is carried separately from registered application
// constraints; current strict composition continues to use its existing rules.
func composeAcquiredScopeWithPolicy(
scope *acquisitionScope,
lower collectionPolicyScope,
policy sequencePolicy,
graph importInclusionGraph,
sources []SourceInfo,
rule *VersionRule,
versionState *sourceVersionState,
accounting *importAccounting,
) (*composedDocument, []Diagnostic) {
traversal := compositionTraversal(policy, rule, versionState, accounting, graph)
if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 {
return nil, diagnostics
}
documents := make([]*document.Node, 0, len(scope.contributions))
for _, contribution := range scope.contributions {
documents = append(documents, contribution.document)
}
composition, diagnostics := composeDocumentsWithCollectionPolicy(
policy, lower, acquiredSequenceDeclarations(*scope), documents...,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil {
return nil, []Diagnostic{*diagnostic}
}
projectNestedEvidence(composition, *scope)
projectAcquiredDestinations(composition, *scope, sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = graph.clone()
return composition, nil
}
func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic {
if scope == nil {
return nil
}
for index := range scope.contributions {
contribution := &scope.contributions[index]
if contribution.pendingItem == nil {
continue
}
placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > limits.nodes {
diagnostic := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)
return &diagnostic
}
accounting.nodes += placementNodes
contribution.placementCharged = true
if diagnostic := reservePendingItemPlacements(contribution.pendingItem, limits, accounting); diagnostic != nil {
return diagnostic
}
}
return nil
}
func (s *importTraversal) visitItemSource(admitted admittedImport, destination, relativeDestination Path, depth int) []Diagnostic {
parentScope := s.scope
parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback
s.scope = acquisitionScope{owner: admitted.nodeKey}
s.itemScope = true
if parentItemScope {
s.itemPrefix = appendDestination(parentPrefix, destination)
} else {
s.itemPrefix = destination.clone()
s.itemFallback = destination.clone()
}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(admitted.source), payload: admitted.payload, document: admitted.payload,
identity: admitted.nodeKey, role: importRoleItem, scopeOwner: admitted.nodeKey,
processing: admitted.processing, sequenceDeclarations: cloneSequenceDeclarations(admitted.sequenceDeclarations),
})
diagnostics := s.walkDeclarations(s.access.dir(admitted.canonical), admitted.declarations, depth, Path{})
itemScope := s.scope
s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback
if len(diagnostics) != 0 {
return diagnostics
}
itemScope.destination = Path{}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(admitted.source), payload: admitted.payload,
identity: admitted.nodeKey, role: importRoleItem, scopeOwner: parentScope.owner,
destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: admitted.processing,
pendingItem: &itemScope,
})
return nil
}
func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnostic {
for index := range scope.contributions {
contribution := &scope.contributions[index]
if contribution.pendingItem == nil {
continue
}
itemScope := contribution.pendingItem
if diagnostics := s.composePendingItems(itemScope); len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index)
}
documents := make([]*document.Node, 0, len(itemScope.contributions))
for _, nested := range itemScope.contributions {
documents = append(documents, nested.document)
}
// An item scope is below a sequence index. Application selectors match
// mapping-only effective paths, so none can apply inside this scope.
// Its YAML declarations remain relative to the owning record.
completed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, itemScope.collectionPolicies,
acquiredSequenceDeclarations(*itemScope), documents...,
)
if len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index)
}
// Version ownership is checked by the enclosing composition once this
// item has its effective coordinates, not against its local field names.
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(*itemScope)}
projectNestedEvidence(completed, *itemScope)
_, placed, err := placeDocumentWithMode(completed.root, contribution.destination, placementModeSequenceItem, contribution.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", contribution.source.ImportedFrom)}
}
placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem)
if !contribution.placementCharged {
if s.state.accounting.nodes+placementNodes > s.policy.importLimits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)}
}
s.state.accounting.nodes += placementNodes
}
contribution.payload = completed.root
contribution.document = placed
contribution.nested = completed
contribution.pendingItem = nil
}
return nil
}
func acquiredSequenceDeclarations(scope acquisitionScope) []sequenceDeclarationSpec {
var declarations []sequenceDeclarationSpec
for _, contribution := range scope.contributions {
projected := cloneSequenceDeclarations(contribution.sequenceDeclarations)
for index := range projected {
projected[index].destination = appendDestination(contribution.relativeDestination, projected[index].destination)
}
declarations = append(declarations, projected...)
}
return declarations
}
func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic {
if composed == nil || rule == nil {
return nil
}
mainIndex := -1
for index, contribution := range scope.contributions {
if contribution.role == importRoleMain {
mainIndex = index
break
}
}
ranges := indexSequenceRanges(composed.sequenceRanges)
for index, contribution := range documents {
if index == mainIndex {
continue
}
if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil {
location := publicLocation(node.Location)
return &Diagnostic{
Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &location,
}
}
}
return nil
}

View File

@@ -139,72 +139,23 @@ func selectDirectoryImportCandidates(access *filesystemAccess, basePath string,
}
entry := frame.entries[frame.next]
frame.next++
accessPath := access.join(frame.directory.accessPath, entry.Name())
lexicalPath := access.join(frame.directory.path, entry.Name())
candidate := selectedImportCandidate{
path: normalizedImportCandidatePath(access, lexicalPath), accessPath: accessPath,
label: relativeImportLabel(access, basePath, lexicalPath), site: frame.directory.site,
result, diagnostic := prepareImportDirectoryEntry(access, basePath, *frame, entry, policy, limits, accounting, activeDirectories)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if !policy.hidden && strings.HasPrefix(entry.Name(), ".") {
switch result.kind {
case importDirectoryEntrySkip:
continue
}
target, entryErr := access.importEntryTarget(candidate.accessPath, entry, policy.follow)
if entryErr != nil {
if errors.Is(entryErr, fs.ErrNotExist) {
case importDirectoryEntryFile:
if _, exists := seenCandidates[result.candidate.path]; exists {
continue
}
return importSelection{}, []Diagnostic{*importDirectoryTargetDiagnostic(frame.directory.site, entryErr)}
seenCandidates[result.candidate.path] = struct{}{}
discovered = append(discovered, result.candidate)
case importDirectoryEntryDirectory:
activeDirectories[result.frame.identity] = struct{}{}
stack = append(stack, result.frame)
}
if target.symlink && !policy.follow {
continue
}
if target.dir {
if !directoryDepthAllows(policy, frame.depth) {
continue
}
if _, exists := activeDirectories[target.identity]; exists {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(frame.directory.site, CodeImportCycle, "directory traversal cycle detected")}
}
childMissing, childIsDirectory, childDiagnostic := validateNamedImportTarget(access, candidate, false, true)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if childMissing || !childIsDirectory {
continue
}
childEntries, vanished, childDiagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if !vanished {
if access.native == nil {
candidate.accessPath = target.identity
}
activeDirectories[target.identity] = struct{}{}
stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries, identity: target.identity})
}
continue
}
if target.identity != "" && access.native == nil {
candidate.accessPath = target.identity
} else {
canonical, canonicalErr := access.canonical(accessPath)
if canonicalErr != nil {
if errors.Is(canonicalErr, fs.ErrNotExist) {
continue
}
return importSelection{}, []Diagnostic{globFilesystemDiagnostic(canonicalErr, "directory search file could not be canonicalized", frame.directory.site.location)}
}
candidate.accessPath = canonical
}
if !matchesImportFilename(entry.Name(), policy.pattern) {
continue
}
if _, exists := seenCandidates[candidate.path]; exists {
continue
}
seenCandidates[candidate.path] = struct{}{}
discovered = append(discovered, candidate)
}
}
sort.Slice(discovered, func(left, right int) bool { return discovered[left].path < discovered[right].path })
@@ -229,6 +180,73 @@ func selectDirectoryImportCandidates(access *filesystemAccess, basePath string,
return selection, nil
}
func prepareImportDirectoryEntry(access *filesystemAccess, basePath string, frame importDirectoryFrame, entry fs.DirEntry, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting, activeDirectories map[string]struct{}) (importDirectoryEntryResult, *Diagnostic) {
accessPath := access.join(frame.directory.accessPath, entry.Name())
lexicalPath := access.join(frame.directory.path, entry.Name())
candidate := selectedImportCandidate{
path: normalizedImportCandidatePath(access, lexicalPath), accessPath: accessPath,
label: relativeImportLabel(access, basePath, lexicalPath), site: frame.directory.site,
}
if !policy.hidden && strings.HasPrefix(entry.Name(), ".") {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
target, err := access.importEntryTarget(candidate.accessPath, entry, policy.follow)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
return importDirectoryEntryResult{}, importDirectoryTargetDiagnostic(frame.directory.site, err)
}
if target.symlink && !policy.follow {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
if target.dir {
if !directoryDepthAllows(policy, frame.depth) {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
if _, exists := activeDirectories[target.identity]; exists {
return importDirectoryEntryResult{}, importSelectionDiagnostic(frame.directory.site, CodeImportCycle, "directory traversal cycle detected")
}
missing, isDirectory, diagnostic := validateNamedImportTarget(access, candidate, false, true)
if diagnostic != nil {
return importDirectoryEntryResult{}, diagnostic
}
if missing || !isDirectory {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
entries, vanished, diagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting)
if diagnostic != nil {
return importDirectoryEntryResult{}, diagnostic
}
if vanished {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
if access.native == nil {
candidate.accessPath = target.identity
}
return importDirectoryEntryResult{kind: importDirectoryEntryDirectory, frame: importDirectoryFrame{
directory: candidate, depth: frame.depth + 1, entries: entries, identity: target.identity,
}}, nil
}
if target.identity != "" && access.native == nil {
candidate.accessPath = target.identity
} else {
canonical, err := access.canonical(accessPath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
diagnostic := globFilesystemDiagnostic(err, "directory search file could not be canonicalized", frame.directory.site.location)
return importDirectoryEntryResult{}, &diagnostic
}
candidate.accessPath = canonical
}
if !matchesImportFilename(entry.Name(), policy.pattern) {
return importDirectoryEntryResult{kind: importDirectoryEntrySkip}, nil
}
return importDirectoryEntryResult{kind: importDirectoryEntryFile, candidate: candidate}, nil
}
type importDirectoryFrame struct {
directory selectedImportCandidate
depth int
@@ -237,6 +255,20 @@ type importDirectoryFrame struct {
identity string
}
type importDirectoryEntryKind uint8
const (
importDirectoryEntrySkip importDirectoryEntryKind = iota
importDirectoryEntryFile
importDirectoryEntryDirectory
)
type importDirectoryEntryResult struct {
kind importDirectoryEntryKind
candidate selectedImportCandidate
frame importDirectoryFrame
}
func importDirectoryTargetDiagnostic(site importFieldSite, err error) *Diagnostic {
if isImportLinkCycle(err) {
return importSelectionDiagnostic(site, CodeImportCycle, "directory traversal cycle detected")

View File

@@ -251,40 +251,17 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo
}
switch entry.Key {
case "required", "recursive", "follow", "hidden":
if entry.Value.Kind != document.Boolean {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, entry.Key+" must be a boolean", entry.Value)}
}
value := entry.Value.Text == "true"
switch entry.Key {
case "required":
spec.required = &value
case "recursive":
spec.recursive = &value
case "follow":
spec.follow = &value
case "hidden":
spec.hidden = &value
if diagnostic := parseImportBooleanAssignment(&spec, entry.Key, entry.Value, fieldPath); diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
case "pattern":
values, origins, ok, badIndex := parseStringSet(entry.Value)
if !ok {
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(entry.Value, fieldPath, "pattern", badIndex)}
if diagnostic := parseImportWildcardAssignment(&spec, entry.Key, entry.Value, fieldPath); diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
if badIndex, ok := invalidWildcard(values); ok {
badPath, badNode := listMemberSite(entry.Value, fieldPath, origins[badIndex])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, "pattern contains an invalid wildcard expression", badNode)}
}
spec.pattern, spec.patternSet = values, true
case "exclude":
values, origins, ok, badIndex := parseStringSet(entry.Value)
if !ok {
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(entry.Value, fieldPath, "exclude", badIndex)}
if diagnostic := parseImportWildcardAssignment(&spec, entry.Key, entry.Value, fieldPath); diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
if badIndex, ok := invalidWildcard(values); ok {
badPath, badNode := listMemberSite(entry.Value, fieldPath, origins[badIndex])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, "exclude contains an invalid wildcard expression", badNode)}
}
spec.exclude, spec.excludeSet = values, true
case "depth":
depth, ok := parseImportDepth(entry.Value)
if !ok {
@@ -298,17 +275,11 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo
}
spec.destination = destination
case "placement":
if entry.Value.Kind != document.String {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
switch entry.Value.Text {
case "mapping":
spec.placement = placementModeMapping
case "sequence-item":
spec.placement = placementModeSequenceItem
default:
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
placement, diagnostic := parseImportPlacement(entry.Value, fieldPath)
if diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
spec.placement = placement
default:
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)}
}
@@ -316,33 +287,98 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo
selectorPath := declarationPath.Key(selectorEntry.Key)
spec.fieldSites[selectorEntry.Key] = importFieldSite{path: selectorPath.clone(), location: publicLocation(selectorEntry.Value.Location)}
if diagnostic := parseImportSelectorValue(&spec, selectorEntry.Key, selectorEntry.Value, selectorPath); diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
return spec, nil
}
func parseImportBooleanAssignment(spec *importDeclarationSpec, field string, node *document.Node, fieldPath Path) *Diagnostic {
if node.Kind != document.Boolean {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, fieldPath, field+" must be a boolean", node)
return &diagnostic
}
value := node.Text == "true"
switch field {
case "required":
spec.required = &value
case "recursive":
spec.recursive = &value
case "follow":
spec.follow = &value
case "hidden":
spec.hidden = &value
}
return nil
}
func parseImportWildcardAssignment(spec *importDeclarationSpec, field string, node *document.Node, fieldPath Path) *Diagnostic {
values, origins, ok, badIndex := parseStringSet(node)
if !ok {
diagnostic := stringSetDiagnostic(node, fieldPath, field, badIndex)
return &diagnostic
}
if badIndex, ok := invalidWildcard(values); ok {
badPath, badNode := listMemberSite(node, fieldPath, origins[badIndex])
diagnostic := nodeImportDiagnostic(CodeImportInvalid, badPath, field+" contains an invalid wildcard expression", badNode)
return &diagnostic
}
switch field {
case "pattern":
spec.pattern, spec.patternSet = values, true
case "exclude":
spec.exclude, spec.excludeSet = values, true
}
return nil
}
func parseImportPlacement(node *document.Node, fieldPath Path) (placementMode, *Diagnostic) {
if node.Kind != document.String {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", node)
return placementModeMapping, &diagnostic
}
switch node.Text {
case "mapping":
return placementModeMapping, nil
case "sequence-item":
return placementModeSequenceItem, nil
default:
diagnostic := nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", node)
return placementModeMapping, &diagnostic
}
}
func parseImportSelectorValue(spec *importDeclarationSpec, field string, node *document.Node, selectorPath Path) *Diagnostic {
switch spec.selector {
case importSelectorPath, importSelectorDirectory:
values, origins, ok, badIndex := parseStringSet(selectorEntry.Value)
values, origins, ok, badIndex := parseStringSet(node)
if !ok {
name := selectorEntry.Key
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(selectorEntry.Value, selectorPath, name, badIndex)}
diagnostic := stringSetDiagnostic(node, selectorPath, field, badIndex)
return &diagnostic
}
for index, value := range values {
if !validImportPath(value) {
badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, selectorEntry.Key+" contains an invalid source path", badNode)}
badPath, badNode := listMemberSite(node, selectorPath, origins[index])
diagnostic := nodeImportDiagnostic(CodeImportInvalid, badPath, field+" contains an invalid source path", badNode)
return &diagnostic
}
badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
badPath, badNode := listMemberSite(node, selectorPath, origins[index])
spec.selectorSites = append(spec.selectorSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
}
spec.paths = values
case importSelectorGlob:
if selectorEntry.Value.Kind != document.String || !validImportPath(selectorEntry.Value.Text) {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob must be a valid nonempty path pattern string", selectorEntry.Value)}
if node.Kind != document.String || !validImportPath(node.Text) {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob must be a valid nonempty path pattern string", node)
return &diagnostic
}
segments, ok := validGlobPattern(selectorEntry.Value.Text)
segments, ok := validGlobPattern(node.Text)
if !ok {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob pattern is invalid", selectorEntry.Value)}
diagnostic := nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob pattern is invalid", node)
return &diagnostic
}
spec.glob, spec.globSegments = selectorEntry.Value.Text, segments
spec.glob, spec.globSegments = node.Text, segments
}
return spec, nil
return nil
}
func selectorForField(field string) importSelector {

File diff suppressed because it is too large Load Diff

View File

@@ -113,10 +113,13 @@ tasks: [{id: z-parent}]
aIndependentPath := writeImportFile(t, directory, "a-independent.yml", "tasks: [{id: a-independent}]\n")
bDependencyPath := writeImportFile(t, directory, "b-dependency.yml", "tasks: [{id: b-dependency}]\n")
input, diagnostics := assembleBaseInput[config](File(mainPath), Options[config]{
Imports: true,
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true}, role: sourceVersionMain,
directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{
sequencePolicy: mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("tasks")}, identityFields: []string{"id"}}}),
})
importLimits: defaultImportLimits(),
}, nil)
if len(diagnostics) != 0 {
t.Fatalf("assemble imports: %#v", diagnostics)
}
@@ -153,7 +156,12 @@ func TestAcquiredContextsUseSelectedFileWhenSourceRecordsAreReordered(t *testing
{ID: "source-2", Path: canonicalTestPath(t, importedPath), Kind: SourceKindFile},
{ID: "source-1", Path: canonicalTestPath(t, mainPath), Kind: SourceKindFile},
}
contexts, diagnostic := contextsForAcquiredSources(File(mainPath), nil, nil, sources)
mainInfo := &SourceInfo{Path: canonicalTestPath(t, mainPath)}
resolved, diagnostic := resolveFilesystemContext(File(mainPath), nil, parsedSource{source: mainInfo}, nil)
if diagnostic != nil {
t.Fatalf("resolve main context: %#v", diagnostic)
}
contexts, diagnostic := contextsForResolvedSources(&resolved, sources)
if diagnostic != nil {
t.Fatalf("source contexts: %#v", diagnostic)
}
@@ -617,9 +625,10 @@ id: item
`)
helperPath := writeImportFile(t, directory, "helper.yml", "value: acquired\n")
accounting := &importAccounting{}
input, diagnostics := assembleBaseInputTracked(File(mainPath), Options[struct{}]{
Imports: true, Filesystem: &FilesystemContext{Root: directory}, deferAssembly: true,
}, Path{}, accounting, nil, nil)
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true, filesystem: &FilesystemContext{Root: directory}},
role: sourceVersionMain, directiveContext: rootDirectiveContextImportsOnly, deferAssembly: true,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, accounting)
if len(diagnostics) != 0 {
t.Fatalf("acquire input: %#v", diagnostics)
}
@@ -674,7 +683,10 @@ func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T)
`)
writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: helper.yml}]}\nname: imported\n")
writeImportFile(t, directory, "helper.yml", "id: helper-id\n")
input, diagnostics := assembleBaseInput(File(mainPath), Options[config]{Imports: true})
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true}, role: sourceVersionMain,
directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, nil)
if len(diagnostics) != 0 {
t.Fatalf("assemble item import: %#v", diagnostics)
}
@@ -951,7 +963,10 @@ func TestItemAcquisitionRetainsInclusionGraphAndScopeOwnership(t *testing.T) {
itemPath := writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: helper.yml}]}\nname: item\n")
helperPath := writeImportFile(t, directory, "helper.yml", "id: supplied\n")
input, diagnostics := assembleBaseInput[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true}, role: sourceVersionMain,
directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, nil)
if len(diagnostics) != 0 {
t.Fatalf("assemble input: %#v", diagnostics)
}
@@ -1104,7 +1119,10 @@ func TestItemHelperScopeScheduleInterleavesUnequalBranches(t *testing.T) {
writeImportFile(t, directory, "b-parent.yml", "configkit: {imports: [{path: a-dependency.yml}]}\nb: true\n")
writeImportFile(t, directory, "a-dependency.yml", "a: true\n")
writeImportFile(t, directory, "c-independent.yml", "c: true\n")
input, diagnostics := assembleBaseInput[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
input, diagnostics := testAcquireInput(inputAcquisitionRequest{
input: loadInput{source: File(mainPath), imports: true}, role: sourceVersionMain,
directiveContext: rootDirectiveContextImportsOnly,
}, acquisitionPolicy{importLimits: defaultImportLimits()}, nil)
if len(diagnostics) != 0 {
t.Fatalf("assemble item scope: %#v", diagnostics)
}
@@ -1268,7 +1286,8 @@ func TestImportTraversalEnforcesPrivateBudgets(t *testing.T) {
{name: "aggregate nodes", limits: importLimits{depth: 8, sources: 8, bytes: maxTotalBytes, nodes: countInputNodes(main.document) + 1}},
} {
t.Run(test.name, func(t *testing.T) {
_, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, test.limits)
request := testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
_, _, diagnostics := testAcquireImportWithLimits(request, test.limits)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("budget diagnostics = %#v", diagnostics)
}
@@ -1278,7 +1297,8 @@ func TestImportTraversalEnforcesPrivateBudgets(t *testing.T) {
writeImportFile(t, directory, "one.yml", "configkit: {imports: [{path: nested.yml}]}\n")
writeImportFile(t, directory, "nested.yml", "settings: {nested: nested}\n")
main = parseImportMain(t, mainPath)
_, _, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, importLimits{depth: 1, sources: 8, bytes: maxTotalBytes, nodes: maxDocumentNodes})
request := testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
_, _, diagnostics := testAcquireImportWithLimits(request, importLimits{depth: 1, sources: 8, bytes: maxTotalBytes, nodes: maxDocumentNodes})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("depth diagnostics = %#v", diagnostics)
}
@@ -1312,7 +1332,9 @@ func TestImportDepthBudgetCountsSelectedEdges(t *testing.T) {
writeImportFile(t, directory, "parts/next.yml", "next: true\n")
limits := defaultImportLimits()
limits.depth = 1
composed, sources, diagnostics := loadImportedDocuments(File(mainPath), nil, parseImportMain(t, mainPath), nil, limits)
main := parseImportMain(t, mainPath)
request := testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
composed, sources, diagnostics := testAcquireImportWithLimits(request, limits)
if test.code != "" {
if len(diagnostics) != 1 || diagnostics[0].Code != test.code {
t.Fatalf("diagnostics = %#v, want %s", diagnostics, test.code)
@@ -1342,7 +1364,8 @@ func TestImportTraversalAcceptsExactPrivateBudgets(t *testing.T) {
bytes: main.bytes + int64(len(data)),
nodes: countInputNodes(main.document) + countInputNodes(imported.document),
}
composed, sources, diagnostics := loadImportedDocuments(File(mainPath), nil, main, nil, limits)
request := testImportRequest(File(mainPath), nil, main, Path{}, nil, nil)
composed, sources, diagnostics := testAcquireImportWithLimits(request, limits)
if len(diagnostics) != 0 || composed == nil || len(sources) != 2 {
t.Fatalf("exact-limit composition = (%#v, %#v, %#v)", composed, sources, diagnostics)
}
@@ -1401,6 +1424,42 @@ func parseImportMain(t *testing.T, filename string) parsedSource {
return parsed
}
func testImportRequest(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, processing *sourceVersionProcessingResult) importAcquisitionRequest {
return importAcquisitionRequest{
input: inputAcquisitionRequest{
input: loadInput{
source: source, imports: true, filesystem: context,
destination: destination.clone(), placementMode: main.source.placementMode,
},
role: sourceVersionMain, directiveContext: rootDirectiveContextImportsOnly,
protectionRule: rule,
},
main: main, processing: processing,
}
}
func testAcquireInput(request inputAcquisitionRequest, policy acquisitionPolicy, accounting *importAccounting) (assembledInput, []Diagnostic) {
if accounting == nil {
accounting = &importAccounting{}
}
state := &acquisitionState{accounting: accounting}
tier := &tierAcquisitionState{visited: make(map[resourceIdentity]importRoute)}
return acquireInput(request, policy, state, tier)
}
func testAcquireImport(request importAcquisitionRequest, policy acquisitionPolicy, state *acquisitionState, tier *tierAcquisitionState) (*composedDocument, []SourceInfo, []Diagnostic) {
composition, sources, _, diagnostics := acquireImportedInput(request, &policy, state, tier)
return composition, sources, diagnostics
}
func testAcquireImportWithLimits(request importAcquisitionRequest, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
return testAcquireImport(request, acquisitionPolicy{importLimits: limits}, &acquisitionState{accounting: &importAccounting{}}, &tierAcquisitionState{})
}
func testAcquireImportWithAccounting(request importAcquisitionRequest, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
return testAcquireImport(request, acquisitionPolicy{importLimits: limits}, &acquisitionState{accounting: accounting}, &tierAcquisitionState{})
}
func parseImportSpecificationsYAML(t *testing.T, input string) []importDeclarationSpec {
t.Helper()
declarations, diagnostics := parseImportSpecificationYAML(input)

View File

@@ -228,17 +228,17 @@ func applyInheritanceSequencePolicy(
}
func applyInheritanceSequenceOrigins(
result *inheritanceScope,
origins, incomingOrigins map[string]Origin,
path Path,
lower, higher *document.Node,
combined sequenceCombinationResult,
operation sequenceMode,
) {
) []inheritanceReplacement {
var replacements []inheritanceReplacement
if operation == sequenceModeReplace {
if lower != nil {
displaced := displacedOrigins(lower, origins, path)
result.replacements = append(result.replacements, inheritanceReplacement{
replacements = append(replacements, inheritanceReplacement{
path: path.clone(), before: lower, after: higher,
previous: displaced, effective: cloneOrigin(incomingOrigins[path.String()]),
})
@@ -253,7 +253,7 @@ func applyInheritanceSequenceOrigins(
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
return
return replacements
}
mergeInheritanceOrigin(origins, path, incomingOrigins[path.String()])
@@ -268,7 +268,7 @@ func applyInheritanceSequenceOrigins(
previous := displacedOrigins(lower.Sequence[replacement.lowerIndex], origins, lowerPath)
deleteOriginSubtree(origins, lowerPath, lower.Sequence[replacement.lowerIndex])
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[replacement.higherIndex], higherPath, outputPath)
result.replacements = append(result.replacements, inheritanceReplacement{
replacements = append(replacements, inheritanceReplacement{
path: outputPath, before: lower.Sequence[replacement.lowerIndex], after: higher.Sequence[replacement.higherIndex],
previous: previous, effective: cloneOrigin(incomingOrigins[higherPath.String()]),
})
@@ -278,6 +278,7 @@ func applyInheritanceSequenceOrigins(
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
return replacements
}
func validateRegisteredInheritanceSequences(root *document.Node, rootPath Path, applicationPolicy sequencePolicy) []Diagnostic {

View File

@@ -0,0 +1,127 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
type inheritanceTemplateResult struct {
value *document.Node
origins map[string]Origin
collectionPolicies collectionPolicyScope
policyTemplates []collectionPolicyScope
controls []inheritanceControl
transfers []inheritanceTransfer
replacements []inheritanceReplacement
operations []inheritanceOperation
}
func combineInheritanceTemplate(
template inheritanceExecutionTemplate,
child preparedInheritanceChild,
parent *preparedInheritanceParent,
parentOrigins map[string]Origin,
applicationPolicy sequencePolicy,
) (inheritanceTemplateResult, []Diagnostic) {
result := inheritanceTemplateResult{}
if child.control != nil {
result.controls = append(result.controls, *child.control)
result.operations = append(result.operations, inheritanceOperation{
control: child.control, controlOrigin: child.controlOrigin,
})
}
if parent == nil {
result.value = document.Clone(child.payload)
result.origins = child.origins
result.collectionPolicies = cloneCollectionPolicyScopeForInheritance(child.policyTemplates)
result.policyTemplates = []collectionPolicyScope{child.policyTemplates}
return result, nil
}
mergedPolicyScope, diagnostics := mergeInheritanceCollectionPolicies(
parent.parentPolicyScope, parent.childPolicyScope,
template.path, parent.payload, child.payload,
applicationPolicy, parent.replaced,
)
if len(diagnostics) != 0 {
return inheritanceTemplateResult{}, diagnostics
}
mergedOrigins := transferredInheritanceOrigins(parentOrigins, parent.transfer.from, template.path, parent.transfer)
mergeInheritanceOrigin(mergedOrigins, template.path, child.origins[template.path.String()])
var sequenceContributions []sequenceHistoryContribution
var combinationDiagnostics []Diagnostic
mergeRecursiveMappings(parent.payload, child.payload, template.path, new(mappingIndex), recursiveMergeHooks{
added: func(childPath Path, incoming *document.Node) {
copyInheritanceSubtree(mergedOrigins, child.origins, incoming, childPath)
},
combined: func(childPath Path, _ *document.Node) {
mergeInheritanceOrigin(mergedOrigins, childPath, child.origins[childPath.String()])
},
replaced: func(childPath Path, existing, incoming *document.Node) {
previous := displacedOrigins(existing, mergedOrigins, childPath)
result.replacements = append(result.replacements, inheritanceReplacement{
path: childPath.clone(), before: existing, after: incoming,
previous: previous, effective: cloneOrigin(child.origins[childPath.String()]),
})
deleteOriginSubtree(mergedOrigins, childPath, existing)
copyInheritanceSubtree(mergedOrigins, child.origins, incoming, childPath)
},
merge: func(childPath Path, existing, incoming *document.Node) (*document.Node, bool) {
resolution, _, _, active, issues := inheritanceCollectionResolution(
childPath, parent.parentPolicyScope, parent.childPolicyScope, applicationPolicy, incoming,
)
if len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
return existing, false
}
if !active {
return nil, false
}
lowerValue := existing
combined, problems := combineSequence(lowerValue, incoming, resolution, sequenceCombinationContext{destination: childPath})
if len(problems) != 0 {
combinationDiagnostics = append(combinationDiagnostics, problems...)
return existing, false
}
policy := applyInheritanceSequencePolicy(
childPath, combined, resolution, parent.parentPolicyScope, parent.childPolicyScope,
)
mergedPolicyScope.set(childPath, policy)
result.replacements = append(result.replacements, applyInheritanceSequenceOrigins(
mergedOrigins, child.origins, childPath, lowerValue, incoming, combined, resolution.operation,
)...)
history := sequenceHistoryContribution{path: childPath.clone(), mode: sequenceHistoryMapped}
if resolution.operation == sequenceModeReplace {
history.mode = sequenceHistoryReplace
if incoming != nil && incoming.Kind == document.Sequence {
history.length = len(incoming.Sequence)
}
} else {
history.items = append([]sequenceItemProjection(nil), combined.higherItems...)
history.replacements = append([]sequenceItemReplacement(nil), combined.replacements...)
}
sequenceContributions = append(sequenceContributions, history)
return combined.value, true
},
})
preserveInheritanceIdentityFields(parent.payload, child.payload, template.identityFields)
if len(combinationDiagnostics) != 0 {
return inheritanceTemplateResult{}, canonicalDiagnostics(combinationDiagnostics)
}
if problems := validateRegisteredInheritanceSequences(parent.payload, template.path, applicationPolicy); len(problems) != 0 {
return inheritanceTemplateResult{}, problems
}
parent.payload.Location = template.entry.Value.Location
result.value = parent.payload
result.origins = mergedOrigins
result.collectionPolicies = inheritancePolicyScopeAtPath(mergedPolicyScope, template.path)
result.policyTemplates = parent.effectivePolicyTemplates
result.transfers = append(result.transfers, parent.transfer)
result.operations = append(result.operations, inheritanceOperation{
transfer: &parent.transfer, parent: parent.historyParent, effective: parent.payload,
parentOrigins: parentOrigins, effectiveOrigins: mergedOrigins,
parentPolicies: cloneInheritancePolicyTemplates(parent.policyTemplates),
templatePolicies: cloneCollectionPolicyScopeForInheritance(child.policyTemplates),
effectivePolicies: cloneInheritancePolicyTemplates(parent.effectivePolicyTemplates),
sequences: cloneSequenceHistoryContributions(sequenceContributions),
identityFields: cloneStrings(template.identityFields),
})
return result, nil
}

View File

@@ -0,0 +1,55 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func chargeInheritanceTemplateHistory(counter *retainedNodeCounter, result inheritanceTemplateResult, template inheritanceExecutionTemplate) *Diagnostic {
for _, operation := range result.operations {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
diagnostic := inheritanceHistoryBudgetDiagnostic(template.path, template.entry.Value.Location)
return &diagnostic
}
}
for _, replacement := range result.replacements {
if !counter.add(replacement.before) || !counter.add(replacement.after) {
diagnostic := inheritanceHistoryBudgetDiagnostic(template.path, template.entry.Value.Location)
return &diagnostic
}
}
return nil
}
func finalizeInheritanceScope(
adapter inheritanceScopeAdapter,
result *inheritanceScope,
templates []inheritanceExecutionTemplate,
resolved []*document.Node,
resolvedOrigins []map[string]Origin,
resolvedPolicies []collectionPolicyScope,
inputOrigins map[string]Origin,
limits projectionLimits,
) []Diagnostic {
root := adapter.rebuild(resolved)
result.collectionPolicies = collectionPolicyScope{}
result.origins = directInheritanceOrigins(&document.Node{Kind: root.Kind, Location: root.Location}, adapter.scopePath(), inputOrigins)
policyIndex := newInheritancePolicyIndex(adapter.collectionPolicies())
for _, slot := range adapter.slots() {
if slot.hasTemplate {
index := slot.template
projectInheritancePolicyRoots(&result.collectionPolicies, resolvedPolicies[index], templates[index].path)
for pointer, origin := range resolvedOrigins[index] {
result.origins[pointer] = origin
}
continue
}
localPolicies := policyIndex.localize(slot.path)
projectInheritancePolicyRoots(&result.collectionPolicies, localPolicies, slot.path)
for pointer, origin := range directInheritanceOrigins(slot.value, slot.path, inputOrigins) {
result.origins[pointer] = origin
}
}
result.root = root
if !inheritanceLowerBoundWithin(result.root, limits.nodes) {
return []Diagnostic{inheritanceBudgetDiagnostic(adapter.scopePath(), adapter.scope().Location)}
}
return nil
}

View File

@@ -0,0 +1,88 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
type preparedInheritanceChild struct {
payload *document.Node
origins map[string]Origin
policyTemplates collectionPolicyScope
control *inheritanceControl
controlOrigin Origin
}
type preparedInheritanceParent struct {
payload *document.Node
historyParent *document.Node
policyTemplates []collectionPolicyScope
effectivePolicyTemplates []collectionPolicyScope
parentPolicyScope collectionPolicyScope
childPolicyScope collectionPolicyScope
replaced collectionReplacementPaths
transfer inheritanceTransfer
}
func prepareInheritanceChild(template inheritanceExecutionTemplate, inputOrigins map[string]Origin) preparedInheritanceChild {
policyTemplates := cloneCollectionPolicyScopeForInheritance(template.collectionPolicies)
payload := withoutInheritanceDirective(template.entry.Value)
prepared := preparedInheritanceChild{
payload: payload, origins: directInheritanceOrigins(payload, template.path, inputOrigins),
policyTemplates: policyTemplates,
}
if template.hasDirective {
control := inheritanceControl{
path: template.directivePath, locations: []Location{publicLocation(template.directive.Value.Location)},
cancelled: template.cancelled,
}
controlOrigins := directInheritanceOrigins(template.directive.Value, control.path, inputOrigins)
prepared.control = &control
prepared.controlOrigin = controlOrigins[control.path.String()]
}
return prepared
}
func prepareInheritanceParent(
template inheritanceExecutionTemplate,
parentPath Path,
parent *document.Node,
resolvedParentPolicyTemplates []collectionPolicyScope,
resolvedParentPolicies collectionPolicyScope,
child preparedInheritanceChild,
applicationPolicy sequencePolicy,
) preparedInheritanceParent {
parentPolicies := cloneInheritancePolicyTemplates(resolvedParentPolicyTemplates)
effectivePolicies := append(
cloneInheritancePolicyTemplates(parentPolicies),
cloneCollectionPolicyScopeForInheritance(child.policyTemplates),
)
parentPolicyScope := projectInheritanceCollectionPolicies(resolvedParentPolicies, template.path)
childPolicyScope := projectCurrentInheritanceCollectionPolicies(child.policyTemplates, template.path)
var replaced collectionReplacementPaths
if len(parentPolicyScope.collections) != 0 || len(childPolicyScope.collections) != 0 {
claims := compositionClaims{byPath: make(map[string][]compositionClaim)}
claims.append(child.payload, template.path, 0)
declarations := make(map[string][]sequenceDeclarationSpec, len(childPolicyScope.collections))
for key, entry := range childPolicyScope.collections {
declarations[key] = inheritanceDeclarationsAt(entry.destination, entry.policy)
}
parentPolicyScope, replaced = lowerCollectionPoliciesForClaims(parentPolicyScope, claims.ordered, declarations, applicationPolicy)
}
parentPayload := inheritanceParentPayload(parent, template.identityFields)
historyParent := parent
if len(template.identityFields) != 0 {
// The retained parent snapshot is the inherited payload, before child
// fields are applied, with parent identity excluded.
historyParent = document.Clone(parentPayload)
}
transfer := inheritanceTransfer{
from: parentPath, to: template.path, reference: template.directivePath,
locations: []Location{publicLocation(template.directive.Value.Location)},
policyTemplates: parentPolicies, templatePolicies: cloneCollectionPolicyScopeForInheritance(child.policyTemplates),
projectedPolicies: projectInheritancePolicyTemplates([]collectionPolicyScope{resolvedParentPolicies}, template.path),
}
return preparedInheritanceParent{
payload: parentPayload, historyParent: historyParent,
policyTemplates: parentPolicies, effectivePolicyTemplates: effectivePolicies,
parentPolicyScope: parentPolicyScope, childPolicyScope: childPolicyScope,
replaced: replaced, transfer: transfer,
}
}

View File

@@ -188,193 +188,52 @@ func applyInheritanceScopeTemplates(adapter inheritanceScopeAdapter, order []int
for _, index := range order {
node := templates[index]
templatePolicies := cloneCollectionPolicyScopeForInheritance(node.collectionPolicies)
payload := withoutInheritanceDirective(node.entry.Value)
operationStart, replacementStart := len(result.operations), len(result.replacements)
childOrigins := directInheritanceOrigins(payload, node.path, inputOrigins)
if node.hasDirective {
control := inheritanceControl{
path: node.directivePath,
locations: []Location{publicLocation(node.directive.Value.Location)},
cancelled: node.cancelled,
}
result.controls = append(result.controls, control)
controlOrigins := directInheritanceOrigins(node.directive.Value, control.path, inputOrigins)
result.operations = append(result.operations, inheritanceOperation{
control: &control, controlOrigin: controlOrigins[control.path.String()],
})
}
child := prepareInheritanceChild(node, inputOrigins)
if limits.nodes >= 0 {
remaining := limits.nodes - retained - adapter.slotOverhead()
within := inheritanceLowerBoundWithin(payload, remaining)
within := inheritanceLowerBoundWithin(child.payload, remaining)
if node.hasParent {
within = inheritanceOverlayLowerBoundWithin(resolved[node.parent], payload, remaining)
within = inheritanceOverlayLowerBoundWithin(resolved[node.parent], child.payload, remaining)
}
if !within {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(node.path, node.entry.Value.Location)}
}
}
var parent *preparedInheritanceParent
var parentOrigins map[string]Origin
if node.hasParent {
parent := resolved[node.parent]
parentPolicies := cloneInheritancePolicyTemplates(resolvedPolicyTemplates[node.parent])
effectivePolicies := append(cloneInheritancePolicyTemplates(parentPolicies), cloneCollectionPolicyScopeForInheritance(templatePolicies))
parentPolicyScope := projectInheritanceCollectionPolicies(resolvedPolicies[node.parent], node.path)
childPolicyScope := projectCurrentInheritanceCollectionPolicies(templatePolicies, node.path)
var replaced collectionReplacementPaths
if len(parentPolicyScope.collections) != 0 || len(childPolicyScope.collections) != 0 {
claims := compositionClaims{byPath: make(map[string][]compositionClaim)}
claims.append(payload, node.path, 0)
declarations := make(map[string][]sequenceDeclarationSpec, len(childPolicyScope.collections))
for key, entry := range childPolicyScope.collections {
declarations[key] = inheritanceDeclarationsAt(entry.destination, entry.policy)
}
parentPolicyScope, replaced = lowerCollectionPoliciesForClaims(parentPolicyScope, claims.ordered, declarations, adapter.applicationPolicy())
}
merged := inheritanceParentPayload(parent, node.identityFields)
historyParent := parent
if len(node.identityFields) != 0 {
// The retained parent snapshot is the inherited payload, before
// child fields are applied, with parent identity excluded.
historyParent = document.Clone(merged)
}
mergedPolicyScope, policyDiagnostics := mergeInheritanceCollectionPolicies(
parentPolicyScope, childPolicyScope, node.path, parent, payload, adapter.applicationPolicy(), replaced,
prepared := prepareInheritanceParent(
node, templates[node.parent].path, resolved[node.parent], resolvedPolicyTemplates[node.parent],
resolvedPolicies[node.parent], child, adapter.applicationPolicy(),
)
if len(policyDiagnostics) != 0 {
return nil, policyDiagnostics
}
transfer := inheritanceTransfer{
from: templates[node.parent].path, to: node.path,
reference: node.directivePath,
locations: []Location{publicLocation(node.directive.Value.Location)},
policyTemplates: parentPolicies, templatePolicies: cloneCollectionPolicyScopeForInheritance(templatePolicies),
projectedPolicies: projectInheritancePolicyTemplates(
[]collectionPolicyScope{resolvedPolicies[node.parent]}, node.path,
),
}
mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[node.parent], transfer.from, node.path, transfer)
mergeInheritanceOrigin(mergedOrigins, node.path, childOrigins[node.path.String()])
var sequenceContributions []sequenceHistoryContribution
var combinationDiagnostics []Diagnostic
mergeRecursiveMappings(merged, payload, node.path, new(mappingIndex), recursiveMergeHooks{
added: func(childPath Path, incoming *document.Node) {
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
},
combined: func(childPath Path, _ *document.Node) {
mergeInheritanceOrigin(mergedOrigins, childPath, childOrigins[childPath.String()])
},
replaced: func(childPath Path, existing, incoming *document.Node) {
previous := displacedOrigins(existing, mergedOrigins, childPath)
result.replacements = append(result.replacements, inheritanceReplacement{
path: childPath.clone(), before: existing, after: incoming,
previous: previous, effective: cloneOrigin(childOrigins[childPath.String()]),
})
deleteOriginSubtree(mergedOrigins, childPath, existing)
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
},
merge: func(childPath Path, existing, incoming *document.Node) (*document.Node, bool) {
resolution, _, _, active, issues := inheritanceCollectionResolution(
childPath, parentPolicyScope, childPolicyScope, adapter.applicationPolicy(), incoming,
)
if len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
return existing, false
}
if !active {
return nil, false
}
lowerValue := existing
combined, problems := combineSequence(lowerValue, incoming, resolution, sequenceCombinationContext{destination: childPath})
if len(problems) != 0 {
combinationDiagnostics = append(combinationDiagnostics, problems...)
return existing, false
}
policy := applyInheritanceSequencePolicy(
childPath, combined, resolution, parentPolicyScope, childPolicyScope,
)
mergedPolicyScope.set(childPath, policy)
applyInheritanceSequenceOrigins(
result, mergedOrigins, childOrigins, childPath, lowerValue, incoming, combined, resolution.operation,
)
history := sequenceHistoryContribution{path: childPath.clone(), mode: sequenceHistoryMapped}
if resolution.operation == sequenceModeReplace {
history.mode = sequenceHistoryReplace
if incoming != nil && incoming.Kind == document.Sequence {
history.length = len(incoming.Sequence)
}
} else {
history.items = append([]sequenceItemProjection(nil), combined.higherItems...)
history.replacements = append([]sequenceItemReplacement(nil), combined.replacements...)
}
sequenceContributions = append(sequenceContributions, history)
return combined.value, true
},
})
preserveInheritanceIdentityFields(merged, payload, node.identityFields)
if len(combinationDiagnostics) != 0 {
return nil, canonicalDiagnostics(combinationDiagnostics)
}
if problems := validateRegisteredInheritanceSequences(merged, node.path, adapter.applicationPolicy()); len(problems) != 0 {
return nil, problems
}
merged.Location = node.entry.Value.Location
resolved[index], resolvedOrigins[index] = merged, mergedOrigins
resolvedPolicies[index] = inheritancePolicyScopeAtPath(mergedPolicyScope, node.path)
result.transfers = append(result.transfers, transfer)
result.operations = append(result.operations, inheritanceOperation{
transfer: &transfer, parent: historyParent, effective: merged,
parentOrigins: resolvedOrigins[node.parent], effectiveOrigins: mergedOrigins,
parentPolicies: cloneInheritancePolicyTemplates(parentPolicies),
templatePolicies: cloneCollectionPolicyScopeForInheritance(templatePolicies),
effectivePolicies: cloneInheritancePolicyTemplates(effectivePolicies),
sequences: cloneSequenceHistoryContributions(sequenceContributions),
identityFields: cloneStrings(node.identityFields),
})
resolvedPolicyTemplates[index] = effectivePolicies
} else {
resolved[index] = document.Clone(payload)
resolvedOrigins[index] = childOrigins
resolvedPolicyTemplates[index] = []collectionPolicyScope{templatePolicies}
resolvedPolicies[index] = cloneCollectionPolicyScopeForInheritance(templatePolicies)
parent = &prepared
parentOrigins = resolvedOrigins[node.parent]
}
templateResult, diagnostics := combineInheritanceTemplate(
node, child, parent, parentOrigins, adapter.applicationPolicy(),
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
resolved[index] = templateResult.value
resolvedOrigins[index] = templateResult.origins
resolvedPolicies[index] = templateResult.collectionPolicies
resolvedPolicyTemplates[index] = templateResult.policyTemplates
result.controls = append(result.controls, templateResult.controls...)
result.transfers = append(result.transfers, templateResult.transfers...)
result.replacements = append(result.replacements, templateResult.replacements...)
result.operations = append(result.operations, templateResult.operations...)
count, _ := inheritanceLowerBoundCount(resolved[index], limits.nodes)
retained += adapter.slotOverhead() + count
for _, operation := range result.operations[operationStart:] {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
return nil, []Diagnostic{inheritanceHistoryBudgetDiagnostic(node.path, node.entry.Value.Location)}
}
}
for _, replacement := range result.replacements[replacementStart:] {
if !counter.add(replacement.before) || !counter.add(replacement.after) {
return nil, []Diagnostic{inheritanceHistoryBudgetDiagnostic(node.path, node.entry.Value.Location)}
}
if diagnostic := chargeInheritanceTemplateHistory(counter, templateResult, node); diagnostic != nil {
return nil, []Diagnostic{*diagnostic}
}
}
root := adapter.rebuild(resolved)
result.collectionPolicies = collectionPolicyScope{}
result.origins = directInheritanceOrigins(&document.Node{Kind: root.Kind, Location: root.Location}, adapter.scopePath(), inputOrigins)
policyIndex := newInheritancePolicyIndex(adapter.collectionPolicies())
for _, slot := range adapter.slots() {
if slot.hasTemplate {
index := slot.template
projectInheritancePolicyRoots(&result.collectionPolicies, resolvedPolicies[index], templates[index].path)
for pointer, origin := range resolvedOrigins[index] {
result.origins[pointer] = origin
}
continue
}
localPolicies := policyIndex.localize(slot.path)
projectInheritancePolicyRoots(&result.collectionPolicies, localPolicies, slot.path)
for pointer, origin := range directInheritanceOrigins(slot.value, slot.path, inputOrigins) {
result.origins[pointer] = origin
}
}
result.root = root
if !inheritanceLowerBoundWithin(result.root, limits.nodes) {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(adapter.scopePath(), adapter.scope().Location)}
if diagnostics := finalizeInheritanceScope(adapter, result, templates, resolved, resolvedOrigins, resolvedPolicies, inputOrigins, limits); len(diagnostics) != 0 {
return nil, diagnostics
}
return result, nil
}

View File

@@ -92,7 +92,7 @@ func TestPendingItemCompositionProjectsNestedIdentityDiagnostics(t *testing.T) {
destination: Keys("items"), pendingItem: &itemScope,
}}}
traversal := importTraversal{accounting: &importAccounting{}, limits: defaultImportLimits()}
traversal := compositionTraversal(sequencePolicy{}, nil, nil, &importAccounting{}, newImportInclusionGraph())
diagnostics := traversal.composePendingItems(&outer)
if len(diagnostics) != 1 {
t.Fatalf("diagnostics = %#v, want one conflict", diagnostics)

View File

@@ -26,19 +26,15 @@ type Definition[T any] struct {
// overlay and requires filesystem context for NoSource. Overrides contain only
// values explicitly supplied by an application CLI.
type Options[T any] struct {
Definition Definition[T]
Imports bool
Overlay string
Filesystem *FilesystemContext
Overrides []Override
SourceVersionMode sourceVersionMode
importDefaults importDefaults
destination Path
placementMode placementMode
sequencePolicy sequencePolicy
deferAssembly bool
directiveContext rootDirectiveContext
importDefaultsValidated bool
Definition Definition[T]
Imports bool
Overlay string
Filesystem *FilesystemContext
Overrides []Override
SourceVersionMode sourceVersionMode
importDefaults importDefaults
destination Path
placementMode placementMode
}
// FilesystemContext supplies filesystem roots for imports and overlays. For an
@@ -153,20 +149,31 @@ func Load[T any](source Source, secrets SecretBinding, supplied ...Options[T]) (
}
func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup secretLookup, supplied ...Options[T]) (*Loaded[T], error) {
options, diagnostics := selectOptions(source, supplied)
policy, policyDiagnostics := compileSequenceIdentityPolicy(options.Definition.sequenceIdentityRegistrations)
diagnostics = append(diagnostics, policyDiagnostics...)
if len(supplied) > 1 {
return nil, (&diagnosticAccumulator{diagnostics: []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "Load accepts at most one options value")}}).loadError()
}
var options Options[T]
if len(supplied) == 1 {
options = supplied[0]
}
input, diagnostics := normalizeInput(loadInput{
source: source, imports: options.Imports, filesystem: options.Filesystem,
destination: options.destination.clone(), placementMode: options.placementMode,
}, options.Overlay)
defaults, policy, settingsDiagnostics := loadSettings(options.importDefaults, options.Definition, options.SourceVersionMode)
diagnostics = append(diagnostics, settingsDiagnostics...)
diagnostics = append(diagnostics, overlayOptionDiagnostics(options.Overlay)...)
if len(diagnostics) != 0 {
return nil, (&diagnosticAccumulator{diagnostics: diagnostics}).loadError()
}
definition := options.Definition
definition.sequenceIdentityRegistrations = nil
request := loadRequest[T]{
main: loadInput{source: source, imports: options.Imports, filesystem: options.Filesystem, destination: options.destination.clone(), placementMode: options.placementMode},
definition: definition, importDefaults: snapshotImportDefaults(options.importDefaults), sequencePolicy: policy, overlay: options.Overlay,
main: input,
definition: definition, importDefaults: defaults, sequencePolicy: policy, overlay: options.Overlay,
overrides: options.Overrides, ordinary: true,
sourceVersionMode: options.SourceVersionMode,
effectiveVersion: effectiveSourceVersionRuleForPlacement(options.Definition.Version, options.destination, options.SourceVersionMode, options.placementMode),
effectiveVersion: effectiveSourceVersionRuleForPlacement(options.Definition.Version, input.destination, options.SourceVersionMode, input.placementMode),
}
return executeLoadRequest(request, secrets, lookup)
}
@@ -174,17 +181,10 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se
func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
accumulator := diagnosticAccumulator{}
accounting := &importAccounting{}
mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode}
mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults)
mainOptions.sequencePolicy = request.sequencePolicy
mainOptions.deferAssembly = true
mainOptions.SourceVersionMode = request.sourceVersionMode
mainOptions.Definition.Version = request.protectionVersion()
if request.ordinary {
mainOptions.Overlay = request.overlay
mainOptions.directiveContext = rootDirectiveContextSource
} else {
mainOptions.directiveContext = rootDirectiveContextLayered
policy := acquisitionPolicy{
importDefaults: snapshotImportDefaults(request.importDefaults),
sequencePolicy: request.sequencePolicy,
importLimits: defaultImportLimits(),
}
var schema *schemaDescriptor
@@ -198,10 +198,20 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
versionState := &sourceVersionState{
localRule: request.definition.Version, effectiveRule: request.protectionVersion(), mode: request.sourceVersionMode,
importDefaults: snapshotImportDefaults(request.importDefaults),
}
state := &acquisitionState{accounting: accounting, versionState: versionState}
accumulator.versionState = versionState
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
directiveContext := rootDirectiveContextLayered
ordinaryOverlay := ""
if request.ordinary {
directiveContext = rootDirectiveContextSource
ordinaryOverlay = request.overlay
}
input, diagnostics := acquireInput(inputAcquisitionRequest{
input: request.main, role: sourceVersionMain, directiveContext: directiveContext,
protectionRule: request.protectionVersion(), deferAssembly: true,
ordinaryOverlay: ordinaryOverlay,
}, policy, state, &tierAcquisitionState{visited: make(map[resourceIdentity]importRoute)})
request.main.source = input.source
accumulator.sources = input.sources
accumulator.add(versionState.diagnostics...)
@@ -219,7 +229,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
}
if request.ordinary && (input.document != nil || input.composition != nil) {
input.composition, diagnostics = compositionForInputWithPolicy(input, request.sequencePolicy)
input.composition, diagnostics = compositionForInputWithPolicy(input, policy.sequencePolicy)
if len(diagnostics) != 0 {
accumulator.add(diagnostics...)
return nil, accumulator.loadError()
@@ -235,10 +245,9 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
if request.ordinary {
versionState.locateMainVersion(input.document)
mainOptions.Definition.Version = versionState.effectiveRule
input, diagnostics = applyInputLayers(request.main.source, mainOptions, input)
input, diagnostics = applyInputLayers(request.main.source, request.main.filesystem, request.overlay, versionState.effectiveRule, policy.sequencePolicy, input)
} else {
input, diagnostics = assembleLayeredInput(request, input, accounting)
input, diagnostics = assembleLayeredInput(request, input, policy, state)
}
accumulator.add(versionState.diagnostics[reportedVersionDiagnostics:]...)
accumulator.sources = input.sources
@@ -255,7 +264,15 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
}
if len(baseSources) != 0 && input.contexts[baseSources[0].ID].FS == nil {
contexts, diagnostic := contextsForAcquiredSources(request.main.source, request.main.filesystem, input.parsed.source, baseSources)
if input.resolvedFilesystem == nil && (request.main.source.kind == sourceFile || request.main.filesystem != nil || nativeContext(request.main.source, request.main.filesystem)) {
resolved, diagnostic := resolveFilesystemContext(request.main.source, request.main.filesystem, input.parsed, nil)
if diagnostic != nil {
accumulator.add(*diagnostic)
return nil, accumulator.loadError()
}
input.resolvedFilesystem = &resolved
}
contexts, diagnostic := contextsForResolvedSources(input.resolvedFilesystem, baseSources)
if diagnostic != nil {
accumulator.add(*diagnostic)
return nil, accumulator.loadError()
@@ -270,7 +287,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
return nil, accumulator.loadError()
}
expanded, problems := expandInheritanceWithCollectionRules(
composition.root, composition.origins, composition.collectionPolicies, request.sequencePolicy, defaultProjectionLimits(),
composition.root, composition.origins, composition.collectionPolicies, policy.sequencePolicy, defaultProjectionLimits(),
)
if len(problems) != 0 {
accumulator.add(problems...)
@@ -342,7 +359,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
if hasErrors(secretResolution.diagnostics) {
return nil, accumulator.loadError()
}
identityDiagnostics := validateFinalSequenceIdentities(resolved.referenceInput, request.sequencePolicy)
identityDiagnostics := validateFinalSequenceIdentities(resolved.referenceInput, policy.sequencePolicy)
accumulator.add(identityDiagnostics...)
if hasErrors(identityDiagnostics) {
return nil, accumulator.loadError()

View File

@@ -1,23 +1,22 @@
package engine
import (
"strconv"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
// assembledInput is the source/document handoff across schema and version checks.
type assembledInput struct {
source Source
parsed parsedSource
document *document.Node
composition *composedDocument
acquisition []acquisitionScope
inheritance *inheritanceScope
sources []SourceInfo
contexts map[string]ResourceContext
versionState *sourceVersionState
accounting *importAccounting
source Source
parsed parsedSource
document *document.Node
composition *composedDocument
acquisition []acquisitionScope
inheritance *inheritanceScope
sources []SourceInfo
contexts map[string]ResourceContext
versionState *sourceVersionState
accounting *importAccounting
resolvedFilesystem *resolvedFilesystemContext
}
// acquiredContribution keeps a source-local payload beside its placed document.
@@ -89,248 +88,42 @@ func cloneAcquisitionScopes(scopes []acquisitionScope) []acquisitionScope {
return cloned
}
func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
return assembleBaseInputTracked(source, options, Path{}, &importAccounting{}, nil, nil)
}
func assembleBaseInputTracked[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
return assembleBaseInputTrackedWithVersion(source, options, destination, accounting, visited, hostToken, nil)
}
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) {
return assembleBaseInputTrackedForRole(source, options, destination, accounting, visited, hostToken, versionState, sourceVersionMain)
}
func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, role sourceVersionRole) (assembledInput, []Diagnostic) {
sourceID := "source-" + strconv.Itoa(accounting.sources+1)
selectedSource := source
if source.kind == sourceSearch {
resolved, failedSource, diagnostics := resolveSearchInput(source, sourceID, defaultDiscoveryDirectoryHelpers(), defaultDiscoveryHostProbe())
if len(diagnostics) != 0 {
if failedSource != nil {
failedSource.Destination = destination.clone()
failedSource.placementMode = options.placementMode
}
return assembledInput{source: resolved, sources: sourceInfos(failedSource), versionState: versionState, accounting: accounting}, diagnostics
}
selectedSource = resolved
if options.Filesystem != nil {
if diagnostics := filesystemOptionDiagnostics(selectedSource, options.Filesystem); len(diagnostics) != 0 {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, diagnostics
}
}
if selectedSource.kind == sourceNone && activeOverlay(options.Overlay) && options.Filesystem == nil {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context")}
}
}
limits := defaultImportLimits()
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
parsed, diagnostics := parseSource(selectedSource, sourceID, parseLimits)
if parsed.source != nil {
parsed.source.Destination = destination.clone()
parsed.source.placementMode = options.placementMode
}
input := assembledInput{source: selectedSource, parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState, accounting: accounting}
if len(diagnostics) != 0 {
input.sources = sourceInfos(parsed.source)
// Mapping value issues already identify application content. Other early
// failures describe source syntax or access, not a placed value.
if selectedSource.kind == sourceMapping && selectedSource.mapIssue != nil && len(destination.segments) != 0 {
for index := range diagnostics {
relative := diagnostics[index].Path
segments := clonePathSegments(destination.segments, len(relative.segments))
diagnostics[index].Path = Path{segments: append(segments, relative.segments...)}
}
}
return input, diagnostics
}
directiveContext := options.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if !options.Imports && parsed.document != nil {
var directiveDiagnostics []Diagnostic
if directiveContext == rootDirectiveContextSource || directiveContext == rootDirectiveContextLayered {
directiveDiagnostics = disabledRootImportDiagnostics(parsed.document)
} else {
directiveDiagnostics = disabledDirectiveDiagnostics(parsed.document, Path{})
}
if len(directiveDiagnostics) != 0 {
return input, directiveDiagnostics
}
}
var versionProcessing *sourceVersionProcessingResult
if directiveContext == rootDirectiveContextSource {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
var processed sourceVersionProcessingResult
var failures []Diagnostic
if versionState != nil {
processed, failures = versionState.processWithDirectiveContext(parsed.document, role, destination, sourceID, options.placementMode, directiveContext)
} else {
processed = processSourceVersionWithDirectiveContext(parsed.document, options.Definition.Version, role, destination, sourceVersionModeDefault, directiveContext)
failures = processed.diagnostics
}
if len(failures) != 0 {
return input, failures
}
versionProcessing = &processed
if role == sourceVersionMain && versionState != nil {
versionState.mainProcessing = &processed
}
} else if directiveContext == rootDirectiveContextLayered {
extracted, failures := extractRootDirectives(parsed.document, rootDirectiveContextLayered)
if len(failures) != 0 {
return input, failures
}
processed := sourceVersionProcessingResult{
payload: extracted.body, imports: extracted.imports, sequences: extracted.sequences,
directiveContext: rootDirectiveContextLayered,
}
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
versionProcessed, versionFailures := versionState.process(extracted.body, role, destination, sourceID, options.placementMode)
if len(versionFailures) != 0 {
return input, versionFailures
}
versionProcessed.imports = extracted.imports
versionProcessed.sequences = append(cloneSequenceDeclarations(extracted.sequences), versionProcessed.sequences...)
versionProcessed.directiveContext = rootDirectiveContextLayered
processed = versionProcessed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
versionProcessing = &processed
} else if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
processed, failures := versionState.process(parsed.document, role, destination, sourceID, options.placementMode)
if len(failures) != 0 {
return input, failures
}
versionProcessing = &processed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
if options.Imports && selectedSource.kind != sourceNone {
if options.deferAssembly {
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessingDeferred(selectedSource, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy, true)
} else {
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(selectedSource, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy)
}
if len(diagnostics) != 0 {
return input, diagnostics
}
if input.composition != nil {
input.acquisition = cloneAcquisitionScopes(input.composition.acquisition)
input.document = input.composition.root
}
} else {
input.sources = sourceInfos(parsed.source)
if versionProcessing != nil {
input.document = versionProcessing.payload
}
if options.Filesystem != nil && (selectedSource.kind == sourceNone || !activeOverlay(options.Overlay)) {
if diagnostic := validateFilesystemContext(selectedSource, options.Filesystem, parsed); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
}
}
if !options.Imports && input.document != nil {
_, placed, err := placeDocumentWithMode(input.document, destination, options.placementMode, input.document.Location, parseLimits.document.Depth, parseLimits.document.Nodes)
if err != nil {
return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
}
localPayload := input.document
input.document = placed
identity := importKeyForMainSource(parsed.source)
var sequenceDeclarations []sequenceDeclarationSpec
if versionProcessing != nil {
sequenceDeclarations = cloneSequenceDeclarations(versionProcessing.sequences)
}
input.acquisition = []acquisitionScope{{
owner: identity,
destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(*parsed.source), payload: localPayload,
identity: identity, role: importRoleMain, scopeOwner: identity,
document: placed, destination: destination.clone(), processing: versionProcessing,
sequenceDeclarations: sequenceDeclarations,
}},
}}
rawNodes := countInputNodes(parsed.document)
placementNodes := destinationWrapperNodeCountForMode(destination, options.placementMode)
if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+rawNodes+placementNodes > limits.nodes {
return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the aggregate input limit")}
}
if diagnostic := registerAcquiredSource(selectedSource, options.Filesystem, parsed.source, visited, hostToken); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
accounting.sources++
accounting.bytes += parsed.bytes
accounting.nodes += rawNodes + placementNodes
}
if !activeOverlay(options.Overlay) || options.Imports {
contexts, diagnostic := contextsForAcquiredSources(selectedSource, options.Filesystem, parsed.source, input.sources)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
input.contexts = contexts
}
return input, nil
}
func registerAcquiredSource(source Source, context *FilesystemContext, info *SourceInfo, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) *Diagnostic {
if visited == nil || info == nil || info.Path == "" {
return nil
}
identity := resourceIdentity{namespace: hostToken, path: info.Path}
if source.kind == sourceFilesystem {
identity.namespace = source.namespace.token
} else if context != nil && context.FS != nil {
identity.namespace = context.FS.namespace.token
}
if previous, repeated := visited[identity]; repeated {
value := importRepeatedDiagnostic(Location{SourceID: info.ID}, previous, importRoute{})
return &value
}
visited[identity] = importRoute{canonical: info.Path}
return nil
}
func applyInputLayers[T any](source Source, options Options[T], input assembledInput) (assembledInput, []Diagnostic) {
if !activeOverlay(options.Overlay) {
func applyInputLayers(source Source, filesystem *FilesystemContext, overlay string, protectionRule *VersionRule, policy sequencePolicy, input assembledInput) (assembledInput, []Diagnostic) {
if !activeOverlay(overlay) {
return input, nil
}
composition, diagnostics := compositionForInputWithPolicy(input, options.sequencePolicy)
composition, diagnostics := compositionForInputWithPolicy(input, policy)
if len(diagnostics) != 0 {
return input, diagnostics
}
assembly := newAssemblyAccumulator(composition)
if activeOverlay(options.Overlay) {
input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(
source, options.Filesystem, input.parsed, assembly, input.sources,
options.Overlay, options.Definition.Version, defaultImportLimits(), input.versionState, options.sequencePolicy,
)
if len(diagnostics) != 0 {
return input, diagnostics
}
contexts, diagnostic := contextsForAcquiredSources(source, options.Filesystem, input.parsed.source, input.sources[len(input.sources)-1:])
if diagnostic != nil {
resolved, diagnostic := resolveOrdinaryOverlayFilesystem(source, filesystem, input.parsed, input.resolvedFilesystem)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
if source.kind != sourceNone || nativeContext(source, filesystem) {
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
mergeResourceContexts(&input.contexts, contexts)
}
request := selectedOverlayRequest{
name: overlay, filesystem: resolved, protectionRule: protectionRule,
sequencePolicy: policy, limits: defaultImportLimits(), versionState: input.versionState,
}
access, diagnostic := openSelectedOverlayAccess(resolved)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
input.sources, diagnostics = loadSelectedOverlay(request, access, assembly, input.sources)
if len(diagnostics) != 0 {
return input, diagnostics
}
input.resolvedFilesystem = &resolved
contexts, diagnostic := contextsForResolvedSources(&resolved, input.sources[len(input.sources)-1:])
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
mergeResourceContexts(&input.contexts, contexts)
input.composition = assembly.current
input.document = assembly.current.root
return input, nil

View File

@@ -49,35 +49,23 @@ type acquiredUnderlayGroup struct {
func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
request.ordinary = false
var diagnostics []Diagnostic
request.importDefaults = snapshotImportDefaults(request.importDefaults)
diagnostics = append(diagnostics, validateImportDefaults(request.importDefaults)...)
policy, policyDiagnostics := compileSequenceIdentityPolicy(request.definition.sequenceIdentityRegistrations)
defaults, policy, diagnostics := loadSettings(request.importDefaults, request.definition, request.sourceVersionMode)
request.importDefaults = defaults
request.sequencePolicy = policy
diagnostics = append(diagnostics, policyDiagnostics...)
request.definition.sequenceIdentityRegistrations = nil
mainOptions, mainDiagnostics := selectOptions(request.main.source, []Options[T]{{
Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(),
SourceVersionMode: request.sourceVersionMode, placementMode: request.main.placementMode,
importDefaults: request.importDefaults, importDefaultsValidated: true,
}})
request.main.destination = mainOptions.destination
main, mainDiagnostics := normalizeInput(request.main, "")
request.main = main
diagnostics = append(diagnostics, mainDiagnostics...)
request.effectiveVersion = effectiveSourceVersionRuleForPlacement(request.definition.Version, request.main.destination, request.sourceVersionMode, request.main.placementMode)
for groupIndex := range request.underlays {
for inputIndex := range request.underlays[groupIndex].inputs {
input := &request.underlays[groupIndex].inputs[inputIndex]
inputOptions, inputDiagnostics := selectOptions(input.source, []Options[T]{{
Imports: input.imports, Filesystem: input.filesystem, destination: input.destination.clone(), placementMode: input.placementMode,
importDefaults: request.importDefaults, importDefaultsValidated: true,
}})
input.destination = inputOptions.destination
normalized, inputDiagnostics := normalizeInput(*input, "")
*input = normalized
diagnostics = append(diagnostics, inputDiagnostics...)
}
}
if !validOverlayName(request.overlay) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "overlay name must contain only ASCII letters, digits, underscore, or hyphen"))
}
diagnostics = append(diagnostics, overlayOptionDiagnostics(request.overlay)...)
if activeOverlay(request.overlay) {
if request.overlayFilesystem == nil {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "selected overlay requires explicit filesystem context"))
@@ -91,14 +79,14 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
return executeLoadRequest(request, secrets, lookup)
}
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, policy acquisitionPolicy, state *acquisitionState) (assembledInput, []Diagnostic) {
var assembly *assemblyAccumulator
mainSourceCount := len(input.sources)
mainEvidence := input.versionState.evidenceSince(0)
projectVersion := sequenceVersionPath(request.protectionVersion())
for index, group := range request.underlays {
evidenceStart := len(input.versionState.evidenceSince(0))
acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), state, policy)
mergeResourceContexts(&input.contexts, acquired.contexts)
if len(diagnostics) != 0 {
input.sources = append(input.sources, acquired.sources...)
@@ -114,7 +102,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
// Source-local checks still run during acquisition above.
protectionRule = nil
}
composition, diagnostics := composeUnderlayGroup(acquired, protectionRule, request.sequencePolicy, lower)
composition, diagnostics := composeUnderlayGroup(acquired, protectionRule, policy.sequencePolicy, lower)
sourceStart := len(input.sources)
input.sources = append(input.sources, acquired.sources...)
if len(diagnostics) != 0 {
@@ -130,7 +118,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
}
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
composition.root, composition.origins, composition.collectionPolicies, composition.applications, LayerUnderlay,
request.sequencePolicy, declarations,
policy.sequencePolicy, declarations,
)
if len(diagnostics) != 0 {
return input, diagnostics
@@ -148,7 +136,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
lower = assembly.current.collectionPolicies
}
var diagnostics []Diagnostic
mainComposition, diagnostics = compositionForInputWithPolicies(input, lower, request.sequencePolicy)
mainComposition, diagnostics = compositionForInputWithPolicies(input, lower, policy.sequencePolicy)
if len(diagnostics) != 0 {
return input, diagnostics
}
@@ -177,7 +165,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
}
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
mainComposition.root, mainComposition.origins, mainComposition.collectionPolicies, mainComposition.applications, LayerBase,
request.sequencePolicy, declarations,
policy.sequencePolicy, declarations,
)
if len(diagnostics) != 0 {
return input, append(versionDiagnostics, diagnostics...)
@@ -201,20 +189,33 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
if assembly == nil {
return input, versionDiagnostics
}
assembly.current.inputBytes = accounting.bytes
assembly.current.inputNodes = accounting.nodes
assembly.current.inputBytes = state.accounting.bytes
assembly.current.inputNodes = state.accounting.nodes
if activeOverlay(request.overlay) {
// The selected overlay context is independent of the main input. This
// synthetic descriptor creates no source record or input contribution.
selection := Source{kind: sourceBytes}
selected := parsedSource{document: &document.Node{Kind: document.Mapping}, source: &SourceInfo{}}
if request.overlayFilesystem == nil {
return input, append(versionDiagnostics, optionDiagnostic(CodeOptionsInvalid, "selected overlay requires explicit filesystem context"))
}
resolved, diagnostic := resolveAbsentFilesystemContext(request.overlayFilesystem)
if diagnostic != nil {
return input, append(versionDiagnostics, *diagnostic)
}
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
return input, append(versionDiagnostics, *diagnostic)
}
overlayRequest := selectedOverlayRequest{
name: request.overlay, filesystem: resolved, protectionRule: request.protectionVersion(),
sequencePolicy: policy.sequencePolicy, limits: defaultImportLimits(), versionState: input.versionState,
}
var diagnostics []Diagnostic
overlayPolicy := []sequencePolicy{request.sequencePolicy}
input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState, overlayPolicy...)
access, diagnostic := openSelectedOverlayAccess(resolved)
if diagnostic != nil {
return input, append(versionDiagnostics, *diagnostic)
}
input.sources, diagnostics = loadSelectedOverlay(overlayRequest, access, assembly, input.sources)
if len(diagnostics) != 0 {
return input, append(versionDiagnostics, diagnostics...)
}
contexts, diagnostic := contextsForAcquiredSources(selection, request.overlayFilesystem, selected.source, input.sources[len(input.sources)-1:])
contexts, diagnostic := contextsForResolvedSources(&resolved, input.sources[len(input.sources)-1:])
if diagnostic != nil {
return input, append(versionDiagnostics, *diagnostic)
}
@@ -225,10 +226,9 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
return input, versionDiagnostics
}
func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) {
initialBytes, initialNodes := accounting.bytes, accounting.nodes
visited := make(map[resourceIdentity]importRoute)
hostToken := &namespaceToken{}
func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, state *acquisitionState, policy acquisitionPolicy) (*acquiredUnderlayGroup, []Diagnostic) {
initialBytes, initialNodes := state.accounting.bytes, state.accounting.nodes
tier := &tierAcquisitionState{visited: make(map[resourceIdentity]importRoute), hostToken: &namespaceToken{}}
nativeAnchors := make(map[*namespaceToken]string)
for inputIndex, contribution := range group.inputs {
if token := nativeTokenForLoadInput(contribution); token != nil {
@@ -240,19 +240,10 @@ func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, a
acquiredGroup := &acquiredUnderlayGroup{index: index}
inclusionGraph := newImportInclusionGraph()
for inputIndex, contribution := range group.inputs {
options := Options[struct{}]{
Imports: contribution.imports, Filesystem: contribution.filesystem,
Definition: Definition[struct{}]{Version: rule}, destination: contribution.destination.clone(),
placementMode: contribution.placementMode, sequencePolicy: policy, deferAssembly: true,
directiveContext: rootDirectiveContextLayered,
importDefaults: func() importDefaults {
if versionState == nil {
return importDefaults{}
}
return snapshotImportDefaults(versionState.importDefaults)
}(),
}
acquired, diagnostics := assembleBaseInputTrackedForRole(contribution.source, options, contribution.destination, accounting, visited, hostToken, versionState, sourceVersionContribution)
acquired, diagnostics := acquireInput(inputAcquisitionRequest{
input: contribution, role: sourceVersionContribution,
directiveContext: rootDirectiveContextLayered, protectionRule: rule, deferAssembly: true,
}, policy, state, tier)
for _, source := range acquired.sources {
source.Layer = LayerUnderlay
source.UnderlayIndex = index
@@ -286,8 +277,8 @@ func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, a
mergeImportInclusionGraph(&inclusionGraph, inputGraph)
}
acquiredGroup.importGraph = inclusionGraph.clone()
acquiredGroup.inputBytes = accounting.bytes - initialBytes
acquiredGroup.inputNodes = accounting.nodes - initialNodes
acquiredGroup.inputBytes = state.accounting.bytes - initialBytes
acquiredGroup.inputNodes = state.accounting.nodes - initialNodes
return acquiredGroup, nil
}

View File

@@ -80,7 +80,8 @@ func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) {
})
accounting := &importAccounting{}
group := loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}
acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, accounting, nil, sequencePolicy{})
state := &acquisitionState{accounting: accounting}
acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, state, acquisitionPolicy{importLimits: defaultImportLimits()})
if len(diagnostics) != 0 {
t.Fatalf("acquire underlay group: %#v", diagnostics)
}
@@ -108,7 +109,7 @@ func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) {
t.Fatalf("composed group handoff = %#v", composition)
}
empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, accounting, nil, sequencePolicy{})
empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, state, acquisitionPolicy{importLimits: defaultImportLimits()})
if len(diagnostics) != 0 || len(empty.scopes) != 0 {
t.Fatalf("empty group acquisition = %#v, %#v", empty, diagnostics)
}
@@ -469,7 +470,8 @@ func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) {
}
func testAcquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, nil, sequencePolicy{})
state := &acquisitionState{accounting: accounting}
acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, state, acquisitionPolicy{importLimits: defaultImportLimits()})
if len(diagnostics) != 0 {
return nil, acquired.sources, acquired.contexts, diagnostics
}

View File

@@ -182,7 +182,12 @@ func TestNativeFilesystemRootAndOptionalImports(t *testing.T) {
t.Fatal(diagnostics)
}
access := newNativeAccess(filesystem, "config", namespace.token)
composed, sources, diagnostics := loadImportedWithAccess(access, "config/sub", source.path, parsed, nil, defaultImportLimits(), &importAccounting{})
request := testImportRequest(source, nil, parsed, Path{}, nil, nil)
policy := &acquisitionPolicy{importLimits: defaultImportLimits()}
state := &acquisitionState{accounting: &importAccounting{}}
tier := &tierAcquisitionState{}
resolved := resolvedFilesystemContext{native: namespace, rootPath: "config", basePath: "config/sub", mainPath: source.path}
composed, sources, diagnostics := acquireImportedWithAccess(request, policy, state, tier, access, resolved)
if len(diagnostics) != 0 || composed == nil || len(sources) != 2 || sources[1].Path != "config/shared.yml" {
t.Fatalf("contained imports = (%#v, %#v, %#v)", composed, sources, diagnostics)
}
@@ -196,7 +201,12 @@ func TestNativeFilesystemRootAndOptionalImports(t *testing.T) {
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, _, diagnostics = loadImportedWithAccess(newNativeAccess(outside, "config", outsideNamespace.token), "config/sub", "config/sub/main.yml", parsed, nil, defaultImportLimits(), &importAccounting{})
request = testImportRequest(filesystemFile(outsideNamespace, "config/sub/main.yml"), nil, parsed, Path{}, nil, nil)
policy = &acquisitionPolicy{importLimits: defaultImportLimits()}
state = &acquisitionState{accounting: &importAccounting{}}
tier = &tierAcquisitionState{}
resolved = resolvedFilesystemContext{native: outsideNamespace, rootPath: "config", basePath: "config/sub", mainPath: "config/sub/main.yml"}
_, _, diagnostics = acquireImportedWithAccess(request, policy, state, tier, newNativeAccess(outside, "config", outsideNamespace.token), resolved)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportOutsideRoot {
t.Fatalf("escaping import diagnostics = %#v", diagnostics)
}

View File

@@ -1,52 +1,52 @@
package engine
func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []Diagnostic) {
if len(supplied) > 1 {
return Options[T]{}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "Load accepts at most one options value")}
}
var options Options[T]
if len(supplied) == 1 {
options = supplied[0]
}
options.importDefaults = snapshotImportDefaults(options.importDefaults)
func normalizeInput(input loadInput, ordinaryOverlay string) (loadInput, []Diagnostic) {
var diagnostics []Diagnostic
if !options.importDefaultsValidated {
diagnostics = validateImportDefaults(options.importDefaults)
}
if options.placementMode != placementModeMapping && options.placementMode != placementModeSequenceItem {
if input.placementMode != placementModeMapping && input.placementMode != placementModeSequenceItem {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "input placement mode is invalid"))
}
if normalized, err := normalizeDestination(options.destination); err != nil {
diagnostics = append(diagnostics, invalidDestinationDiagnostic(options.destination))
if normalized, err := normalizeDestination(input.destination); err != nil {
diagnostics = append(diagnostics, invalidDestinationDiagnostic(input.destination))
} else {
options.destination = normalized
input.destination = normalized
}
if options.placementMode == placementModeSequenceItem && len(options.destination.segments) == 0 {
if input.placementMode == placementModeSequenceItem && len(input.destination.segments) == 0 {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "sequence-item placement requires a nonempty destination"))
}
definition := options.Definition
if !validOverlayName(options.Overlay) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "overlay name must contain only ASCII letters, digits, underscore, or hyphen"))
if input.filesystem != nil {
diagnostics = append(diagnostics, filesystemOptionDiagnostics(input.source, input.filesystem)...)
}
if options.Filesystem != nil {
diagnostics = append(diagnostics, filesystemOptionDiagnostics(source, options.Filesystem)...)
}
if source.kind == sourceSearch {
for _, issue := range source.searchIssues {
if input.source.kind == sourceSearch {
for _, issue := range input.source.searchIssues {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, issue))
}
if source.search == nil {
if input.source.search == nil {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "configuration search is invalid"))
} else {
diagnostics = append(diagnostics, validateSearch(*source.search, source.searchApp)...)
diagnostics = append(diagnostics, validateSearch(*input.source.search, input.source.searchApp)...)
}
}
if source.kind == sourceNone && activeOverlay(options.Overlay) && options.Filesystem == nil {
if input.source.kind == sourceNone && activeOverlay(ordinaryOverlay) && input.filesystem == nil {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context"))
}
if options.Filesystem == nil && (source.kind == sourceBytes || source.kind == sourceReader || source.kind == sourceMapping) && (options.Imports || activeOverlay(options.Overlay)) {
if input.filesystem == nil && (input.source.kind == sourceBytes || input.source.kind == sourceReader || input.source.kind == sourceMapping) && (input.imports || activeOverlay(ordinaryOverlay)) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "synthetic-source imports or overlays require filesystem context"))
}
return input, canonicalDiagnostics(diagnostics)
}
func loadSettings[T any](defaults importDefaults, definition Definition[T], mode sourceVersionMode) (importDefaults, sequencePolicy, []Diagnostic) {
defaults = snapshotImportDefaults(defaults)
var diagnostics []Diagnostic
diagnostics = append(diagnostics, validateImportDefaults(defaults)...)
diagnostics = append(diagnostics, sourceVersionOptionDiagnostics(definition, mode)...)
policy, policyDiagnostics := compileSequenceIdentityPolicy(definition.sequenceIdentityRegistrations)
diagnostics = append(diagnostics, policyDiagnostics...)
return defaults, policy, canonicalDiagnostics(diagnostics)
}
func sourceVersionOptionDiagnostics[T any](definition Definition[T], mode sourceVersionMode) []Diagnostic {
var diagnostics []Diagnostic
if definition.Version != nil {
if !definition.Version.Path.valid() {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "schema-version path is invalid"))
@@ -55,12 +55,19 @@ func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []D
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "schema-version rule requires a checker"))
}
}
if options.SourceVersionMode != sourceVersionModeDefault && options.SourceVersionMode != sourceVersionModeIfPresent && options.SourceVersionMode != sourceVersionModeRequired {
if mode != sourceVersionModeDefault && mode != sourceVersionModeIfPresent && mode != sourceVersionModeRequired {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "source-version mode is invalid"))
} else if options.SourceVersionMode != sourceVersionModeDefault && (definition.Version == nil || !definition.Version.Path.valid() || definition.Version.Check == nil) {
} else if mode != sourceVersionModeDefault && (definition.Version == nil || !definition.Version.Path.valid() || definition.Version.Check == nil) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "source-version checking requires a valid version rule and checker"))
}
return options, canonicalDiagnostics(diagnostics)
return diagnostics
}
func overlayOptionDiagnostics(name string) []Diagnostic {
if validOverlayName(name) {
return nil
}
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "overlay name must contain only ASCII letters, digits, underscore, or hyphen")}
}
func optionDiagnostic(code Code, message string) Diagnostic {

View File

@@ -39,89 +39,9 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor
switch descriptor.kind {
case descriptorStruct:
result, issue := o.node(document.Mapping, path, depth)
if issue != nil {
return nil, issue
}
result.Mapping = make([]document.Entry, 0, len(descriptor.fields))
seen := make(map[string]bool, len(descriptor.fields))
for _, member := range descriptor.members {
memberValue := value.Field(member.index)
if member.inline {
if isNilValue(memberValue) {
continue
}
inline, issue := o.project(memberValue, member.descriptor, path, depth)
if issue != nil {
return nil, issue
}
if inline.Kind != document.Mapping {
return nil, &projectionIssue{code: CodeSchemaUnsupportedType, path: path.clone(), message: "inline value did not project to a mapping"}
}
o.nodes-- // The flattened container is not retained in the output tree.
for _, entry := range inline.Mapping {
if seen[entry.Key] {
return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: path.Key(entry.Key), message: "inline value conflicts with a declared field"}
}
seen[entry.Key] = true
result.Mapping = append(result.Mapping, entry)
}
continue
}
childPath := path.Key(member.name)
if seen[member.name] {
return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: childPath, message: "inline value conflicts with a declared field"}
}
if issue := o.charge(childPath); issue != nil { // mapping key
return nil, issue
}
child, issue := o.project(memberValue, member.descriptor, childPath, depth+1)
if issue != nil {
return nil, issue
}
seen[member.name] = true
result.Mapping = append(result.Mapping, document.Entry{Key: member.name, Value: child})
}
for _, entry := range result.Mapping {
field := descriptor.inlineMap
if index, ok := descriptor.fieldByName[entry.Key]; ok {
field = &descriptor.fields[index]
}
if field != nil && field.secret {
entry.Value.Sensitive = true
}
}
return result, nil
return o.projectStruct(value, descriptor, path, depth)
case descriptorMap:
if value.IsNil() {
return o.node(document.Null, path, depth)
}
leave, issue := o.enter(value, path)
if issue != nil {
return nil, issue
}
defer leave()
result, issue := o.node(document.Mapping, path, depth)
if issue != nil {
return nil, issue
}
result.Mapping = make([]document.Entry, 0, value.Len())
keys := value.MapKeys()
slices.SortFunc(keys, func(left, right reflect.Value) int {
return cmp.Compare(left.String(), right.String())
})
for _, key := range keys {
childPath := path.Key(key.String())
if issue := o.charge(childPath); issue != nil {
return nil, issue
}
child, issue := o.project(value.MapIndex(key), descriptor.element, childPath, depth+1)
if issue != nil {
return nil, issue
}
result.Mapping = append(result.Mapping, document.Entry{Key: key.String(), Value: child})
}
return result, nil
return o.projectMap(value, descriptor, path, depth)
case descriptorSlice:
if value.IsNil() {
return o.node(document.Null, path, depth)
@@ -131,22 +51,9 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor
return nil, issue
}
defer leave()
fallthrough
return o.projectSequence(value, descriptor, path, depth)
case descriptorArray:
result, issue := o.node(document.Sequence, path, depth)
if issue != nil {
return nil, issue
}
result.Sequence = make([]*document.Node, 0, value.Len())
for i := 0; i < value.Len(); i++ {
childPath := path.Index(i)
child, issue := o.project(value.Index(i), descriptor.element, childPath, depth+1)
if issue != nil {
return nil, issue
}
result.Sequence = append(result.Sequence, child)
}
return result, nil
return o.projectSequence(value, descriptor, path, depth)
case descriptorDynamic:
if value.IsNil() {
return o.node(document.Null, path, depth)
@@ -217,6 +124,111 @@ func (o *valueOperation) project(value reflect.Value, descriptor *typeDescriptor
}
}
func (o *valueOperation) projectStruct(value reflect.Value, descriptor *typeDescriptor, path Path, depth int) (*document.Node, *projectionIssue) {
result, issue := o.node(document.Mapping, path, depth)
if issue != nil {
return nil, issue
}
result.Mapping = make([]document.Entry, 0, len(descriptor.fields))
seen := make(map[string]bool, len(descriptor.fields))
for _, member := range descriptor.members {
memberValue := value.Field(member.index)
if member.inline {
if isNilValue(memberValue) {
continue
}
inline, issue := o.project(memberValue, member.descriptor, path, depth)
if issue != nil {
return nil, issue
}
if inline.Kind != document.Mapping {
return nil, &projectionIssue{code: CodeSchemaUnsupportedType, path: path.clone(), message: "inline value did not project to a mapping"}
}
o.nodes-- // The flattened container is not retained in the output tree.
for _, entry := range inline.Mapping {
if seen[entry.Key] {
return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: path.Key(entry.Key), message: "inline value conflicts with a declared field"}
}
seen[entry.Key] = true
result.Mapping = append(result.Mapping, entry)
}
continue
}
childPath := path.Key(member.name)
if seen[member.name] {
return nil, &projectionIssue{code: CodeSchemaAmbiguousField, path: childPath, message: "inline value conflicts with a declared field"}
}
if issue := o.charge(childPath); issue != nil { // mapping key
return nil, issue
}
child, issue := o.project(memberValue, member.descriptor, childPath, depth+1)
if issue != nil {
return nil, issue
}
seen[member.name] = true
result.Mapping = append(result.Mapping, document.Entry{Key: member.name, Value: child})
}
for _, entry := range result.Mapping {
field := descriptor.inlineMap
if index, ok := descriptor.fieldByName[entry.Key]; ok {
field = &descriptor.fields[index]
}
if field != nil && field.secret {
entry.Value.Sensitive = true
}
}
return result, nil
}
func (o *valueOperation) projectMap(value reflect.Value, descriptor *typeDescriptor, path Path, depth int) (*document.Node, *projectionIssue) {
if value.IsNil() {
return o.node(document.Null, path, depth)
}
leave, issue := o.enter(value, path)
if issue != nil {
return nil, issue
}
defer leave()
result, issue := o.node(document.Mapping, path, depth)
if issue != nil {
return nil, issue
}
result.Mapping = make([]document.Entry, 0, value.Len())
keys := value.MapKeys()
slices.SortFunc(keys, func(left, right reflect.Value) int {
return cmp.Compare(left.String(), right.String())
})
for _, key := range keys {
childPath := path.Key(key.String())
if issue := o.charge(childPath); issue != nil {
return nil, issue
}
child, issue := o.project(value.MapIndex(key), descriptor.element, childPath, depth+1)
if issue != nil {
return nil, issue
}
result.Mapping = append(result.Mapping, document.Entry{Key: key.String(), Value: child})
}
return result, nil
}
func (o *valueOperation) projectSequence(value reflect.Value, descriptor *typeDescriptor, path Path, depth int) (*document.Node, *projectionIssue) {
result, issue := o.node(document.Sequence, path, depth)
if issue != nil {
return nil, issue
}
result.Sequence = make([]*document.Node, 0, value.Len())
for index := 0; index < value.Len(); index++ {
childPath := path.Index(index)
child, issue := o.project(value.Index(index), descriptor.element, childPath, depth+1)
if issue != nil {
return nil, issue
}
result.Sequence = append(result.Sequence, child)
}
return result, nil
}
func (o *valueOperation) node(kind document.Kind, path Path, depth int) (*document.Node, *projectionIssue) {
if issue := o.checkDepth(path, depth); issue != nil {
return nil, issue

View File

@@ -237,22 +237,6 @@ func SourceInfoImportSelectionsForFacade(source SourceInfo) []ImportSelectionFor
// SourceInfoWithImportSelections copies converted root-owned records into
// private engine source evidence.
func SourceInfoWithImportSelections(source SourceInfo, selections []ImportSelectionForFacade) SourceInfo {
if selections == nil {
source.importSelections = nil
return source
}
source.importSelections = make([]importSelectionEvidence, len(selections))
for index, selection := range selections {
source.importSelections[index] = importSelectionEvidence{
path: selection.Path.clone(), location: selection.Location,
selector: importSelectorFromString(selection.Selector),
outcome: importSelectionOutcomeFromString(selection.Outcome),
}
}
return source
}
func importSelectorString(value importSelector) string {
switch value {
case importSelectorPath:
@@ -266,19 +250,6 @@ func importSelectorString(value importSelector) string {
}
}
func importSelectorFromString(value string) importSelector {
switch value {
case "path":
return importSelectorPath
case "dir":
return importSelectorDirectory
case "glob":
return importSelectorGlob
default:
return 0
}
}
func importSelectionOutcomeString(value importSelectionOutcome) string {
if value == importSelectionNone {
return "none"
@@ -286,13 +257,6 @@ func importSelectionOutcomeString(value importSelectionOutcome) string {
return ""
}
func importSelectionOutcomeFromString(value string) importSelectionOutcome {
if value == "none" {
return importSelectionNone
}
return 0
}
// SourceInfoPlacementModeForFacade converts private placement evidence to its
// root-owned numeric representation.
func SourceInfoPlacementModeForFacade(source SourceInfo) uint8 {
@@ -304,13 +268,6 @@ func SourceInfoDestinationUnresolvedForFacade(source SourceInfo) bool {
return source.destinationUnresolved
}
// SourceInfoWithPlacement copies facade-owned source evidence into private fields.
func SourceInfoWithPlacement(source SourceInfo, mode uint8, unresolved bool) SourceInfo {
source.placementMode = placementMode(mode)
source.destinationUnresolved = unresolved
return source
}
// SourceVersionCheck is a detached, public-safe record of a source-local
// version declaration observed during an enabled load.
type SourceVersionCheck struct {

View File

@@ -241,80 +241,15 @@ func (b *schemaBuilder) describeFields(descriptor *typeDescriptor, current refle
if name != "" {
return &schemaIssue{code: CodeSchemaTag, path: path.clone(), message: "inline field cannot declare a YAML name"}
}
inlineType := field.Type
seenPointers := make(map[reflect.Type]bool)
for inlineType.Kind() == reflect.Pointer {
if seenPointers[inlineType] {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline pointer chain must terminate in a struct"}
}
seenPointers[inlineType] = true
inlineType = inlineType.Elem()
}
if inlineType.Kind() == reflect.Map && field.Type.Kind() == reflect.Pointer {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline maps cannot be pointers"}
}
switch inlineType.Kind() {
case reflect.Struct:
if b.building[inlineType] {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "recursive inline struct is not supported"}
}
child, childIssue := b.describe(inlineType, path)
if childIssue != nil {
return childIssue
}
if child.kind != descriptorStruct {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a structural struct or string-keyed map"}
}
for _, nested := range child.fields {
nested.index = prependIndex(i, nested.index)
nested.secret = secret || nested.secret
if issue := addField(descriptor, nested, path); issue != nil {
return issue
}
}
if child.inlineMap != nil {
nested := *child.inlineMap
nested.index = prependIndex(i, nested.index)
nested.secret = secret || nested.secret
if descriptor.inlineMap != nil {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"}
}
descriptor.inlineMap = &nested
}
memberType, memberIssue := b.describe(field.Type, path)
if memberIssue != nil {
return memberIssue
}
descriptor.members = append(descriptor.members, structMember{index: i, descriptor: memberType, inline: true})
case reflect.Map:
if inlineType.Key().Kind() != reflect.String {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline map must have string keys"}
}
if descriptor.inlineMap != nil {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"}
}
fieldPath := path.Key("*")
fieldType, childIssue := b.describe(inlineType, fieldPath)
if childIssue != nil {
return childIssue
}
descriptor.inlineMap = &fieldDescriptor{index: []int{i}, descriptor: fieldType, secret: secret, inlineMap: true}
descriptor.members = append(descriptor.members, structMember{index: i, descriptor: fieldType, inline: true})
default:
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a struct, pointer to struct, or string-keyed map"}
if issue := b.describeInlineField(descriptor, field, i, secret, path); issue != nil {
return issue
}
continue
}
fieldPath := path.Key(name)
fieldType, childIssue := b.describe(field.Type, fieldPath)
if childIssue != nil {
return childIssue
}
if issue := addField(descriptor, fieldDescriptor{name: name, index: []int{i}, descriptor: fieldType, secret: secret}, path); issue != nil {
if issue := b.describeOrdinaryField(descriptor, field, i, name, secret, path); issue != nil {
return issue
}
descriptor.members = append(descriptor.members, structMember{index: i, name: name, descriptor: fieldType})
}
return nil
}

View File

@@ -0,0 +1,102 @@
package engine
import "reflect"
func (b *schemaBuilder) describeOrdinaryField(owner *typeDescriptor, field reflect.StructField, index int, name string, secret bool, path Path) *schemaIssue {
fieldPath := path.Key(name)
fieldType, issue := b.describe(field.Type, fieldPath)
if issue != nil {
return issue
}
if issue := addField(owner, fieldDescriptor{name: name, index: []int{index}, descriptor: fieldType, secret: secret}, path); issue != nil {
return issue
}
owner.members = append(owner.members, structMember{index: index, name: name, descriptor: fieldType})
return nil
}
func (b *schemaBuilder) describeInlineField(owner *typeDescriptor, field reflect.StructField, index int, secret bool, path Path) *schemaIssue {
inlineType, issue := inlineFieldType(field.Type, path)
if issue != nil {
return issue
}
if inlineType.Kind() == reflect.Map && field.Type.Kind() == reflect.Pointer {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline maps cannot be pointers"}
}
switch inlineType.Kind() {
case reflect.Struct:
return b.describeInlineStruct(owner, field, inlineType, index, secret, path)
case reflect.Map:
return b.describeInlineMap(owner, inlineType, index, secret, path)
default:
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a struct, pointer to struct, or string-keyed map"}
}
}
func inlineFieldType(fieldType reflect.Type, path Path) (reflect.Type, *schemaIssue) {
current := fieldType
seenPointers := make(map[reflect.Type]bool)
for current.Kind() == reflect.Pointer {
if seenPointers[current] {
return nil, &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline pointer chain must terminate in a struct"}
}
seenPointers[current] = true
current = current.Elem()
}
return current, nil
}
func (b *schemaBuilder) describeInlineStruct(owner *typeDescriptor, field reflect.StructField, inlineType reflect.Type, index int, secret bool, path Path) *schemaIssue {
if b.building[inlineType] {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "recursive inline struct is not supported"}
}
child, issue := b.describe(inlineType, path)
if issue != nil {
return issue
}
if child.kind != descriptorStruct {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline field must be a structural struct or string-keyed map"}
}
for _, nested := range child.fields {
nested.index = prependIndex(index, nested.index)
nested.secret = secret || nested.secret
if issue := addField(owner, nested, path); issue != nil {
return issue
}
}
if child.inlineMap != nil {
nested := *child.inlineMap
nested.index = prependIndex(index, nested.index)
nested.secret = secret || nested.secret
if owner.inlineMap != nil {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"}
}
owner.inlineMap = &nested
}
memberType, issue := b.describe(field.Type, path)
if issue != nil {
return issue
}
owner.members = append(owner.members, structMember{index: index, descriptor: memberType, inline: true})
return nil
}
func (b *schemaBuilder) describeInlineMap(owner *typeDescriptor, inlineType reflect.Type, index int, secret bool, path Path) *schemaIssue {
if inlineType.Key().Kind() != reflect.String {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "inline map must have string keys"}
}
if owner.inlineMap != nil {
return &schemaIssue{code: CodeSchemaInline, path: path.clone(), message: "configuration object has more than one inline map"}
}
fieldPath := path.Key("*")
fieldType, issue := b.describe(inlineType, fieldPath)
if issue != nil {
return issue
}
owner.inlineMap = &fieldDescriptor{index: []int{index}, descriptor: fieldType, secret: secret, inlineMap: true}
owner.members = append(owner.members, structMember{index: index, descriptor: fieldType, inline: true})
return nil
}

View File

@@ -28,84 +28,51 @@ func validOverlayName(name string) bool {
return true
}
func loadSelectedOverlayDocument(source Source, context *FilesystemContext, main parsedSource, base *composedDocument, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
assembly := newAssemblyAccumulator(cloneComposedDocument(base))
allSources, diagnostics := loadSelectedOverlayInto(source, context, main, assembly, sources, name, rule, limits)
if len(diagnostics) != 0 {
return nil, allSources, diagnostics
}
return assembly.current, allSources, nil
type selectedOverlayRequest struct {
name string
filesystem resolvedFilesystemContext
protectionRule *VersionRule
sequencePolicy sequencePolicy
limits importLimits
versionState *sourceVersionState
}
func loadSelectedOverlayInto(source Source, context *FilesystemContext, main parsedSource, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) {
return loadSelectedOverlayIntoWithVersion(source, context, main, base, sources, name, rule, limits, nil)
}
func loadSelectedOverlayIntoWithVersion(source Source, context *FilesystemContext, main parsedSource, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState, policies ...sequencePolicy) ([]SourceInfo, []Diagnostic) {
var policy sequencePolicy
if len(policies) != 0 {
policy = policies[0]
}
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil {
return cloneSourceInfos(sources), []Diagnostic{*diagnostic}
}
if diagnostic := validateNativeDirectories(access, basePath, mainPath); diagnostic != nil {
return cloneSourceInfos(sources), []Diagnostic{*diagnostic}
}
if len(policies) == 0 {
return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState)
}
return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState, policy)
}
var rootPath string
func resolveOrdinaryOverlayFilesystem(source Source, context *FilesystemContext, main parsedSource, existing *resolvedFilesystemContext) (resolvedFilesystemContext, *Diagnostic) {
if source.kind == sourceNone {
if context == nil {
value := optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context")
return cloneSourceInfos(sources), []Diagnostic{value}
}
var err error
rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return cloneSourceInfos(sources), []Diagnostic{value}
}
} else {
var diagnostic *Diagnostic
rootPath, _, _, diagnostic = importFilesystem(source, context, main)
if diagnostic != nil {
return cloneSourceInfos(sources), []Diagnostic{*diagnostic}
return resolvedFilesystemContext{}, &value
}
return resolveAbsentFilesystemContext(context)
}
access, err := openHostAccess(rootPath)
if existing != nil {
return *existing, nil
}
return resolveFilesystemContext(source, context, main, nil)
}
func openSelectedOverlayAccess(resolved resolvedFilesystemContext) (*filesystemAccess, *Diagnostic) {
access, err := openResolvedFilesystemAccess(resolved)
if err != nil {
value := globFilesystemDiagnostic(err, "configuration root could not be opened for the selected overlay", Location{})
value.Primary = nil
return cloneSourceInfos(sources), []Diagnostic{value}
return nil, &value
}
if len(policies) == 0 {
return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState)
}
return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, versionState, policy)
return access, nil
}
func loadSelectedOverlayWithAccess(access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) {
return loadSelectedOverlayWithAccessAndVersion(access, base, sources, name, rule, limits, nil)
}
func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits, versionState *sourceVersionState, policies ...sequencePolicy) ([]SourceInfo, []Diagnostic) {
var policy sequencePolicy
if len(policies) != 0 {
policy = policies[0]
}
func loadSelectedOverlay(request selectedOverlayRequest, access *filesystemAccess, base *assemblyAccumulator, sources []SourceInfo) ([]SourceInfo, []Diagnostic) {
policy := request.sequencePolicy
rule := request.protectionRule
limits := request.limits
versionState := request.versionState
closed := false
defer func() {
if !closed {
_ = access.close()
}
}()
overlayPath := access.join(access.rootPath, "overlays/"+name+".yml")
overlayPath := access.join(access.rootPath, "overlays/"+request.name+".yml")
canonical, err := access.canonical(overlayPath)
if err != nil {
code := CodeOverlayInvalid
@@ -138,7 +105,7 @@ func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *ass
sourceID := "source-" + strconv.Itoa(len(sources)+1)
overlayLocation := &Location{SourceID: sourceID}
kind := SourceKindFile
label := "overlays/" + name + ".yml"
label := "overlays/" + request.name + ".yml"
if access.native != nil {
kind = SourceKindFilesystem
label = access.label + "/" + canonical
@@ -160,66 +127,44 @@ func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *ass
if base.current.inputNodes+nodeCount > limits.nodes {
return sources, []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the aggregate node limit", overlayLocation)}
}
if diagnostics := overlayDirectiveDiagnostics(overlayRoot, Path{}, len(policies) != 0); len(diagnostics) != 0 {
if diagnostics := overlayDirectiveDiagnostics(overlayRoot, Path{}, true); len(diagnostics) != 0 {
return sources, diagnostics
}
var processed sourceVersionProcessingResult
var failures []Diagnostic
if len(policies) != 0 {
if versionState != nil {
processed, failures = versionState.processWithDirectiveContext(
overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping, rootDirectiveContextOverlay,
)
} else {
processed = processSourceVersionWithDirectiveContext(
overlayRoot, rule, sourceVersionContribution, Path{}, sourceVersionModeDefault, rootDirectiveContextOverlay,
)
failures = processed.diagnostics
}
if versionState != nil {
processed, failures = versionState.processWithDirectiveContext(
overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping, rootDirectiveContextOverlay,
)
} else {
if versionState != nil && versionState.mode != sourceVersionModeDefault {
processed, failures = versionState.process(overlayRoot, sourceVersionContribution, Path{}, sourceID, placementModeMapping)
} else {
processed = sourceVersionProcessingResult{payload: overlayRoot}
}
processed = processSourceVersionWithDirectiveContext(
overlayRoot, rule, sourceVersionContribution, Path{}, sourceVersionModeDefault, rootDirectiveContextOverlay,
)
failures = processed.diagnostics
}
if len(failures) != 0 {
return sources, failures
}
overlayPayload := processed.payload
if versionNode := fragmentVersionClaim(overlayPayload, rule); len(policies) == 0 && versionNode != nil {
location := publicLocation(versionNode.Location)
return sources, []Diagnostic{{
Code: CodeOverlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "overlays cannot contribute to or replace the schema-version path", Primary: &location,
}}
}
closeErr := access.close()
closed = true
if closeErr != nil {
return sources, []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed after reading the selected overlay", nil)}
}
var overlayDiagnostics []Diagnostic
if len(policies) != 0 {
overlayDiagnostics = base.applyOverlayWithCollectionPolicy(overlayPayload, nil, collectionPolicyScope{}, nil, LayerOverlay, policy, processed.sequences)
} else {
overlayDiagnostics = base.applyOverlay(overlayPayload, nil, nil, LayerOverlay)
}
overlayDiagnostics := base.applyOverlayWithCollectionPolicy(overlayPayload, nil, collectionPolicyScope{}, nil, LayerOverlay, policy, processed.sequences)
if len(overlayDiagnostics) != 0 {
return sources, overlayDiagnostics
}
if len(policies) != 0 {
// The selected overlay retains one application with the actual item
// assignments. Raw list indexes cannot identify an upsert's destination.
application := base.current.applications[len(base.current.applications)-1]
if node := versionClaimFromApplication(application, rule); node != nil {
location := publicLocation(node.Location)
return sources, []Diagnostic{{
Code: CodeOverlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "overlays cannot contribute to or replace the schema-version path", Primary: &location,
}}
}
// The selected overlay retains one application with the actual item
// assignments. Raw list indexes cannot identify an upsert's destination.
application := base.current.applications[len(base.current.applications)-1]
if node := versionClaimFromApplication(application, rule); node != nil {
location := publicLocation(node.Location)
return sources, []Diagnostic{{
Code: CodeOverlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "overlays cannot contribute to or replace the schema-version path", Primary: &location,
}}
}
base.current.inputBytes += int64(len(data))
base.current.inputNodes += nodeCount

View File

@@ -585,7 +585,21 @@ func TestLoadOverlayCountsAgainstAggregateBudgets(t *testing.T) {
}
base.inputBytes = main.bytes
base.inputNodes = countInputNodes(main.document)
_, _, diagnostics = loadSelectedOverlayDocument(File(mainPath), nil, main, base, sourceInfos(main.source), "testing", nil, test.limits(main, len(overlayData)))
resolved, diagnostic := resolveFilesystemContext(File(mainPath), nil, main, nil)
if diagnostic != nil {
t.Fatal(diagnostic)
}
request := selectedOverlayRequest{
name: "testing", filesystem: resolved, limits: test.limits(main, len(overlayData)),
}
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
t.Fatal(diagnostic)
}
access, diagnostic := openSelectedOverlayAccess(resolved)
if diagnostic != nil {
t.Fatal(diagnostic)
}
_, diagnostics = loadSelectedOverlay(request, access, newAssemblyAccumulator(base), sourceInfos(main.source))
if len(diagnostics) == 0 || diagnostics[0].Code != CodeOverlayLimitExceeded {
t.Fatalf("diagnostics = %#v", diagnostics)
}

View File

@@ -7,62 +7,18 @@ import (
"strings"
)
func contextsForAcquiredSources(source Source, context *FilesystemContext, baseSource *SourceInfo, sources []SourceInfo) (map[string]ResourceContext, *Diagnostic) {
func contextsForResolvedSources(resolved *resolvedFilesystemContext, sources []SourceInfo) (map[string]ResourceContext, *Diagnostic) {
if len(sources) == 0 {
return nil, nil
}
var rootPath, basePath string
var filesystem fs.FS
if source.kind == sourceFilesystem {
filesystem = source.namespace.filesystem
rootPath, basePath = ".", path.Dir(source.path)
if context != nil && context.Root != "" {
rootPath = context.Root
}
} else if context != nil && context.FS != nil {
filesystem = context.FS.namespace.filesystem
rootPath, basePath = context.Root, context.BaseDir
} else if source.kind == sourceFile {
filePath := source.path
if baseSource != nil {
filePath = baseSource.Path
} else {
canonical, err := canonicalExistingPath(filePath)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "selected source could not be canonicalized", nil)
return nil, &value
}
filePath = canonical
}
basePath = filepath.Dir(filePath)
rootPath = basePath
if context != nil && context.Root != "" {
canonical, err := canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return nil, &value
}
rootPath = canonical
}
} else if context != nil {
var err error
rootPath, err = canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return nil, &value
}
if source.kind == sourceNone && context.BaseDir == "" {
basePath = rootPath
} else {
basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return nil, &value
}
}
} else {
if resolved == nil {
return nil, nil
}
rootPath, basePath := resolved.rootPath, resolved.basePath
var filesystem fs.FS
if resolved.native != nil {
filesystem = resolved.native.filesystem
}
var rooted fs.FS
if filesystem != nil {
rooted = rootedNativeFS{filesystem: filesystem, root: rootPath}
@@ -81,7 +37,7 @@ func contextsForAcquiredSources(source Source, context *FilesystemContext, baseS
}
var relative string
if filesystem != nil {
if !fs.ValidPath(rootPath) || !fs.ValidPath(base) || rootPath != "." && base != rootPath && !strings.HasPrefix(base, rootPath+"/") {
if !containsNativePath(rootPath, base) {
value := importDiagnostic(CodeImportOutsideRoot, "source base is outside the configuration root", nil)
return nil, &value
}

View File

@@ -47,7 +47,6 @@ type sourceVersionState struct {
localRule *VersionRule
effectiveRule *VersionRule
mode sourceVersionMode
importDefaults importDefaults
mainProcessing *sourceVersionProcessingResult
evidence []*sourceVersionEvidence
diagnostics []Diagnostic

View File

@@ -1487,9 +1487,22 @@ func TestSourceVersionOverlayProtectsMountedMainAndPreservesSequenceCoordinates(
t.Fatalf("compose base: %#v", compositionDiagnostics)
}
assembly := newAssemblyAccumulator(base)
_, diagnostics = loadSelectedOverlayIntoWithVersion(
File(mainPath), nil, main, assembly, sourceInfos(main.source), "testing", effectiveRule, defaultImportLimits(), state,
)
resolved, diagnostic := resolveFilesystemContext(File(mainPath), nil, main, nil)
if diagnostic != nil {
t.Fatalf("resolve overlay context: %#v", diagnostic)
}
request := selectedOverlayRequest{
name: "testing", filesystem: resolved, protectionRule: effectiveRule,
limits: defaultImportLimits(), versionState: state,
}
if diagnostic := validateResolvedFilesystemDirectories(resolved); diagnostic != nil {
t.Fatalf("validate overlay context: %#v", diagnostic)
}
access, diagnostic := openSelectedOverlayAccess(resolved)
if diagnostic != nil {
t.Fatalf("open overlay context: %#v", diagnostic)
}
_, diagnostics = loadSelectedOverlay(request, access, assembly, sourceInfos(main.source))
if len(diagnostics) != 0 {
t.Fatalf("load selected overlay: %#v", diagnostics)
}
@@ -1523,9 +1536,11 @@ func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T)
limits.nodes = wantNodes
accounting := &importAccounting{}
composed, sources, foundDiagnostics := loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, limits, accounting, nil, nil, state,
)
request := testImportRequest(File(mainPath), nil, main, Path{}, rule, nil)
policy := acquisitionPolicy{importLimits: limits}
owners := &acquisitionState{accounting: accounting, versionState: state}
tier := &tierAcquisitionState{}
composed, sources, foundDiagnostics := testAcquireImport(request, policy, owners, tier)
if len(foundDiagnostics) != 0 || composed == nil || len(sources) != 2 || accounting.nodes != wantNodes || accounting.bytes != wantBytes {
t.Fatalf("exact raw byte/node accounting = %#v, %d/%d, want %d/%d, %#v", composed, accounting.bytes, accounting.nodes, wantBytes, wantNodes, foundDiagnostics)
}
@@ -1539,10 +1554,11 @@ func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T)
below := defaultImportLimits()
below.nodes = wantNodes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
request = testImportRequest(File(mainPath), nil, main, Path{}, rule, nil)
policy.importLimits = below
owners = &acquisitionState{accounting: accounting, versionState: &sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired}}
tier = &tierAcquisitionState{}
composed, _, foundDiagnostics = testAcquireImport(request, policy, owners, tier)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header nodes were not charged: %#v, %#v", composed, foundDiagnostics)
}
@@ -1550,10 +1566,11 @@ func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T)
below = defaultImportLimits()
below.bytes = wantBytes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
request = testImportRequest(File(mainPath), nil, main, Path{}, rule, nil)
policy.importLimits = below
owners = &acquisitionState{accounting: accounting, versionState: &sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired}}
tier = &tierAcquisitionState{}
composed, _, foundDiagnostics = testAcquireImport(request, policy, owners, tier)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header bytes were not charged: %#v, %#v", composed, foundDiagnostics)
}

View File

@@ -88,13 +88,6 @@ func toEngineLayeredOptions[T any](options LayeredOptions[T]) engine.LayeredOpti
}
}
}
if options.Overrides != nil {
result.Overrides = make([]engine.Override, len(options.Overrides))
for index, override := range options.Overrides {
result.Overrides[index] = engine.Override{
Path: toEnginePath(override.Path), Value: override.Value, Flag: override.Flag,
}
}
}
result.Overrides = toEngineOverrides(options.Overrides)
return result
}

View File

@@ -323,14 +323,7 @@ func toEngineOptions[T any](options Options[T]) engine.Options[T] {
if options.Filesystem != nil {
result.Filesystem = toEngineFilesystemContext(options.Filesystem)
}
if options.Overrides != nil {
result.Overrides = make([]engine.Override, len(options.Overrides))
for index := range options.Overrides {
result.Overrides[index] = engine.Override{
Path: toEnginePath(options.Overrides[index].Path), Value: options.Overrides[index].Value, Flag: options.Overrides[index].Flag,
}
}
}
result.Overrides = toEngineOverrides(options.Overrides)
return result
}

View File

@@ -1,6 +1,8 @@
package configkit
import (
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/engine"
@@ -27,16 +29,11 @@ func TestOverlayPublicValues(t *testing.T) {
func TestSourceInfoDestinationConversionsAndCopies(t *testing.T) {
destination := Keys("collections", "a/b", "0")
public := SourceInfo{
ID: "source-1", Destination: destination,
PlacementMode: PlacementModeSequenceItem, DestinationUnresolved: true,
}
internal := toEngineSourceInfo(public)
if internal.Destination.String() != destination.String() || engine.SourceInfoPlacementModeForFacade(internal) != uint8(PlacementModeSequenceItem) || !engine.SourceInfoDestinationUnresolvedForFacade(internal) {
t.Fatalf("engine source evidence = %#v", internal)
}
path := writeRecordConfig(t, "value: main\n")
options := engine.OptionsWithPlacement(engine.Options[map[string]any]{}, engine.Keys("collections", "a/b", "0"), uint8(PlacementModeSequenceItem))
internal := engineSourceInfoFromLoad(t, path, "source-1", options)
converted := fromEngineSourceInfo(internal)
if converted.Destination.String() != destination.String() || converted.PlacementMode != PlacementModeSequenceItem || !converted.DestinationUnresolved {
if converted.Destination.String() != destination.String() || converted.PlacementMode != PlacementModeSequenceItem || converted.DestinationUnresolved {
t.Fatalf("public source evidence = %#v", converted)
}
copy := cloneSourceInfo(converted)
@@ -47,58 +44,116 @@ func TestSourceInfoDestinationConversionsAndCopies(t *testing.T) {
}
func TestSourceVersionCheckConversionsAndCopies(t *testing.T) {
path := Keys("records").Index(2).Key("version")
source := SourceInfo{
ID: "source-1",
VersionCheck: &SourceVersionCheck{
Path: path, Version: Version{Present: true, Text: "123"},
Location: &Location{SourceID: "source-1", Line: 4, Column: 8},
path := Keys("version")
file := writeRecordConfig(t, "version: '123'\nvalue: main\n")
options := engine.Options[recordConfig]{
Definition: engine.Definition[recordConfig]{
Version: &engine.VersionRule{
Path: engine.Keys("version"),
Check: func(version engine.Version, _ *engine.Validation) error {
if !version.Present || version.Text != "123" {
t.Fatalf("version callback input = %#v", version)
}
return nil
},
},
},
SourceVersionMode: engine.SourceVersionModeFromFacade(uint8(SourceVersionModeIfPresent)),
}
internal := toEngineSourceInfo(source)
if internal.VersionCheck == nil || internal.VersionCheck.Path.String() != path.String() ||
internal.VersionCheck.Version.Text != "123" || internal.VersionCheck.Location == nil || internal.VersionCheck.Location.Line != 4 {
t.Fatalf("engine source version check = %#v", internal.VersionCheck)
}
internal := engineSourceInfoFromLoad(t, file, "source-1", options)
converted := fromEngineSourceInfo(internal)
if converted.VersionCheck == nil || converted.VersionCheck.Path.String() != path.String() ||
!converted.VersionCheck.Version.Present || converted.VersionCheck.Version.Text != "123" ||
converted.VersionCheck.Location == nil || converted.VersionCheck.Location.SourceID != internal.ID || converted.VersionCheck.Location.Line != 1 {
t.Fatalf("public source version check = %#v", converted.VersionCheck)
}
copy := cloneSourceInfo(converted)
copy.VersionCheck.Path = Keys("changed")
copy.VersionCheck.Version.Text = "changed"
copy.VersionCheck.Location.Line = 99
copy.VersionCheck.Redacted = true
if converted.VersionCheck.Path.String() != path.String() || converted.VersionCheck.Version.Text != "123" ||
converted.VersionCheck.Location.Line != 4 || converted.VersionCheck.Redacted {
converted.VersionCheck.Location.Line != 1 || converted.VersionCheck.Redacted {
t.Fatalf("source version check changed through a detached copy: %#v", converted.VersionCheck)
}
}
func TestImportSelectionConversionsAndCopies(t *testing.T) {
declaration, err := ParsePath("/configkit/imports/2")
if err != nil {
t.Fatal(err)
}
source := SourceInfo{ID: "source-4", ImportSelections: []ImportSelection{{
Path: declaration, Location: Location{SourceID: "source-4", Line: 7, Column: 5},
Selector: ImportSelectorGlob, Outcome: ImportSelectionNone,
}}}
internal := toEngineSourceInfo(source)
transport := engine.SourceInfoImportSelectionsForFacade(internal)
if len(transport) != 1 || transport[0].Path.String() != declaration.String() || transport[0].Location.SourceID != source.ID || transport[0].Selector != "glob" || transport[0].Outcome != "none" {
t.Fatalf("engine selection evidence = %#v", transport)
}
source.ImportSelections[0].Path = Keys("changed")
file := writeRecordConfig(t, "configkit:\n imports:\n - glob: absent/*.yml\nvalue: main\n")
internal := engineSourceInfoFromLoad(t, file, "source-1", engine.Options[recordConfig]{Imports: true})
converted := fromEngineSourceInfo(internal)
if len(converted.ImportSelections) != 1 || converted.ImportSelections[0].Path.String() != declaration.String() || converted.ImportSelections[0].Location.Line != 7 || converted.ImportSelections[0].Selector != ImportSelectorGlob || converted.ImportSelections[0].Outcome != ImportSelectionNone {
if len(converted.ImportSelections) != 1 || converted.ImportSelections[0].Path.String() != "/configkit/imports/0" ||
converted.ImportSelections[0].Location.SourceID != internal.ID || converted.ImportSelections[0].Location.Line != 3 ||
converted.ImportSelections[0].Selector != ImportSelectorGlob || converted.ImportSelections[0].Outcome != ImportSelectionNone {
t.Fatalf("converted selection evidence = %#v", converted.ImportSelections)
}
copy := cloneSourceInfo(converted)
copy.ImportSelections[0].Path = Keys("copy")
copy.ImportSelections[0].Selector = ImportSelectorPath
if converted.ImportSelections[0].Path.String() != declaration.String() || converted.ImportSelections[0].Selector != ImportSelectorGlob {
if converted.ImportSelections[0].Path.String() != "/configkit/imports/0" || converted.ImportSelections[0].Selector != ImportSelectorGlob {
t.Fatalf("source selection evidence changed through copy: %#v", converted.ImportSelections)
}
if converted.ImportSelections[0].Outcome != ImportSelectionNone {
t.Fatalf("source selection outcome = %q", converted.ImportSelections[0].Outcome)
}
if fromEngineImportSelections(nil) != nil {
t.Fatal("converting nil engine selections returned a non-nil slice")
}
if fromEngineImportSelections([]engine.ImportSelectionForFacade{}) == nil {
t.Fatal("converting an empty engine selection slice returned nil")
}
empty := cloneSourceInfo(SourceInfo{ImportSelections: make([]ImportSelection, 0)})
if empty.ImportSelections == nil {
t.Fatal("cloning collapsed a non-nil empty selection slice")
}
}
func TestOverrideConversionsPreserveSliceShapeAndCopyRecords(t *testing.T) {
if toEngineOverrides(nil) != nil {
t.Fatal("nil overrides converted to a non-nil slice")
}
empty := make([]Override, 0)
if toEngineOverrides(empty) == nil || toEngineOptions(Options[recordConfig]{Overrides: empty}).Overrides == nil ||
toEngineLayeredOptions(LayeredOptions[recordConfig]{Overrides: empty}).Overrides == nil {
t.Fatal("a non-nil empty override slice was collapsed")
}
overrides := []Override{{Path: Keys("feature", "enabled"), Value: "yes", Flag: "--feature-enabled"}}
ordinary := toEngineOptions(Options[recordConfig]{Overrides: overrides})
layered := toEngineLayeredOptions(LayeredOptions[recordConfig]{Overrides: overrides})
overrides[0] = Override{Path: Keys("changed"), Value: "no", Flag: "--changed"}
for label, converted := range map[string][]engine.Override{"ordinary": ordinary.Overrides, "layered": layered.Overrides} {
if len(converted) != 1 || converted[0].Path.String() != "/feature/enabled" || converted[0].Value != "yes" || converted[0].Flag != "--feature-enabled" {
t.Errorf("%s overrides = %#v", label, converted)
}
}
}
type recordConfig struct {
Value string `yaml:"value"`
Version string `yaml:"version"`
}
func writeRecordConfig(t *testing.T, content string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return path
}
func engineSourceInfoFromLoad[T any](t *testing.T, path, sourceID string, options engine.Options[T]) engine.SourceInfo {
t.Helper()
loaded, err := engine.Load(engine.File(path), engine.SecretBinding{}, options)
if err != nil {
t.Fatalf("engine load: %v", err)
}
for _, source := range loaded.Inspection.Sources() {
if source.ID == sourceID {
return source
}
}
t.Fatalf("engine inspection contains no source %q", sourceID)
return engine.SourceInfo{}
}