159 Commits

Author SHA1 Message Date
7ff204bf3d Clarify v0.9.0 import and collection migration guidance 2026-10-24 14:48:21 +00:00
b3ec860fac Document the ConfigKit v0.9.0 release 2026-10-24 14:44:58 +00:00
435dea102f Retire the completed remediation record 2026-10-24 14:41:56 +00:00
ea46001f78 Reuse document indexes for collection policy checks 2026-10-24 14:17:09 +00:00
a0e5693d07 Share indexed fallback locations during effective projection 2026-10-24 14:13:52 +00:00
5c5f2af6cc Limit inheritance policy work to the relevant scope 2026-10-24 14:10:44 +00:00
8253993005 Delete CLI metadata through the replaced subtree 2026-10-24 14:05:42 +00:00
d0848290da Keep sequence projection storage proportional to assigned items 2026-10-24 14:04:26 +00:00
e94a0642ff Keep global registrations out of item-local composition 2026-10-24 13:59:34 +00:00
571d063675 Retire the completed sequence roadmap and implementation plan 2026-10-24 13:56:19 +00:00
d0a605afbb Document the internal complexity refactor roadmap 2026-10-24 02:02:43 +00:00
7a83704f43 Clarify discovery contexts and filename selection boundaries 2026-10-24 02:00:17 +00:00
ac5bf29ddc Settle discovery slug and custom path rules 2026-10-24 01:58:48 +00:00
cce8356b8e Settle empty discovery searches and filename selections 2026-10-24 01:55:02 +00:00
b746dcc609 Settle discovery behavior when no main file is found 2026-10-24 01:51:08 +00:00
9c32b8d3b5 Record sequence algorithm audit findings 2026-10-24 01:46:55 +00:00
c73daf891d Settle discovery directory resolution errors 2026-10-24 01:39:25 +00:00
ae038a5165 Settle discovery presets and directory helpers 2026-10-24 01:33:53 +00:00
367bbaaa1b Clarify sequence operation history in the configuration contract 2026-10-24 01:21:11 +00:00
fafc28421f Build item policies without repeated accumulated copies 2026-10-24 01:20:12 +00:00
1a673753b9 Preserve child identity in inheritance history 2026-10-24 01:17:00 +00:00
988e49bb09 Define discovery filename groups and search presets 2026-10-24 01:16:42 +00:00
5bb4f364b7 Track the main version through sequence movement 2026-10-24 01:11:28 +00:00
af84107905 Document composable configuration discovery 2026-10-24 01:00:08 +00:00
f94dd9b961 Order CLI metadata history after inheritance 2026-10-24 00:55:21 +00:00
146e7e56d9 Retire obsolete collection identity before replacement agreement 2026-10-24 00:51:10 +00:00
d2e20a304a Record sequence feature integration audit findings 2026-10-24 00:38:04 +00:00
8d40983271 Avoid repeated prefix scans when projecting item policies 2026-10-24 00:19:14 +00:00
25cf7ce05c Clarify final identity validation and overlay semantics 2026-10-24 00:16:53 +00:00
57edb8ec19 Identify conflicting sequence registration positions 2026-10-24 00:15:23 +00:00
42663cf07c Retain main version warnings in layered loads 2026-10-24 00:13:54 +00:00
085b57eeb6 Validate final identity fields as application payload 2026-10-24 00:12:31 +00:00
dcf0379e02 Apply nested collection operations when copying mappings 2026-10-24 00:11:21 +00:00
8ecfedc91b Reset inherited YAML identity on actual replacement 2026-10-24 00:07:22 +00:00
f7624ae92a Project incoming source destinations through tier operations 2026-10-24 00:05:24 +00:00
a035df9845 Preserve tier sequence operations in source history 2026-10-24 00:01:46 +00:00
e6598810e0 Project sequence origins from incoming to final item paths 2026-10-23 23:51:33 +00:00
f8cbb2ed87 Preserve retained input presence in nested CLI overrides 2026-10-23 23:47:21 +00:00
af80ccb90c Record sequence implementation review findings 2026-10-23 23:30:27 +00:00
0be7cdb00e Audit sequence identity scaling and lifecycle 2026-10-23 22:03:16 +00:00
e01aeca761 Reconcile sequence identity documentation 2026-10-23 21:57:35 +00:00
d4133f23f5 Add integrated sequence catalog examples 2026-10-23 21:50:10 +00:00
5af71a7533 Validate final registered sequence identities 2026-10-23 21:43:54 +00:00
41b5dff69b Apply CLI overrides after typed decoding 2026-10-23 21:37:13 +00:00
ecfb78d84d Prepare post-decode CLI inspection context 2026-10-23 21:20:19 +00:00
041c4cff02 Apply CLI overrides to decoded configuration 2026-10-23 21:11:54 +00:00
23f22d8982 Prepare reusable CLI override batches 2026-10-23 21:02:46 +00:00
0023617a07 Expose sequence identity registrations through the facade 2026-10-23 20:59:23 +00:00
33a51606da Resolve inheritance between identified records 2026-10-23 20:48:46 +00:00
268fd03579 Prepare reusable inheritance scope execution 2026-10-23 20:32:10 +00:00
9d4d2e2a17 Build identified record inheritance graphs 2026-10-23 20:23:09 +00:00
36ff9170b7 Apply collection policies during sibling inheritance 2026-10-23 20:19:05 +00:00
9e75496664 Prepare collection policy transport through inheritance 2026-10-23 20:04:06 +00:00
80d8c1dc82 Preserve nested collection policy ownership 2026-10-23 19:56:06 +00:00
acd9e9a9fa Integrate collection policies across layered assembly 2026-10-23 19:49:12 +00:00
89a9684f0d Activate selected overlay collection policies 2026-10-23 19:30:48 +00:00
9a7027a04c Activate ordinary source collection declarations 2026-10-23 19:12:39 +00:00
dfeaf90320 Integrate policy-aware item scopes 2026-10-23 18:55:03 +00:00
d03e56398b Resolve collection identity during strict composition 2026-10-23 18:39:04 +00:00
628fc53d2c Separate underlay acquisition from composition 2026-10-23 18:22:37 +00:00
5fec16a768 Separate input acquisition from composition 2026-10-23 18:15:01 +00:00
f6b02dec03 Project sequence evidence through item mappings 2026-10-23 18:07:15 +00:00
7b04dc6a50 Extend history replay for mapped sequence items 2026-10-23 17:55:45 +00:00
12fdf0d331 Implement shared sequence operations 2026-10-23 17:49:05 +00:00
b202dd187f Track collection policy ownership 2026-10-23 14:16:22 +00:00
0314e587b6 Reconcile collection declarations and modes 2026-10-23 14:07:07 +00:00
9d7635d1e4 Parse collection sequence declarations 2026-10-23 14:01:39 +00:00
298e70b552 Add dynamic sequence identity selectors 2026-10-23 13:55:25 +00:00
64c51647f0 Add shared sequence identity primitives 2026-10-23 13:49:22 +00:00
bc6b977585 Split the sequence implementation into smaller stages 2026-10-23 13:40:29 +00:00
d06aae5461 Plan sequence identity and combination implementation 2026-10-23 12:56:23 +00:00
4f941f8ba4 Clarify sequence policy interactions and rename the roadmap 2026-10-23 12:20:18 +00:00
9a49833339 Remove the completed operator convention audit 2026-10-23 12:17:29 +00:00
749cf6868c Settle the sequence identity registration API 2026-10-23 12:09:44 +00:00
a5f0b92ee9 Add dynamic identity constraints to the sequence roadmap 2026-10-23 01:56:00 +00:00
6ec23b2de1 Extend collection inheritance architecture to identified sequences 2026-10-23 01:42:32 +00:00
3298eb916c Settle how sequence inheritance references compose 2026-10-23 01:40:37 +00:00
40430e787c Add identified record inheritance to the sequence roadmap 2026-10-23 01:31:22 +00:00
f434fa159f Align architecture policy with collection and CLI decisions 2026-10-23 01:12:11 +00:00
8ddf73ede3 Accept collection policy and CLI lifecycle decisions 2026-10-23 01:09:24 +00:00
844a15a3cb Consolidate the sequence roadmap around the settled design 2026-10-23 00:11:50 +00:00
58f4c39d6f Settle sequence identity declarations without values 2026-10-23 00:04:55 +00:00
8a54cecd1d Settle nested sequence policy ownership and record review findings 2026-10-23 00:02:26 +00:00
0e430e8077 Allow lower-tier identity during strict composition 2026-10-22 23:53:48 +00:00
26caa6024e Settle persistent sequence operation policy 2026-10-22 23:47:50 +00:00
8691a93ebf Settle CLI ordering before application resolution 2026-10-22 23:41:06 +00:00
c563f80934 Record remaining sequence lifecycle decisions 2026-10-22 15:45:21 +00:00
27797d2392 Settle sequence inheritance and lifecycle coverage 2026-10-22 15:16:42 +00:00
6a4e57e5d3 Settle sequence validation and absent value behavior 2026-10-22 14:47:00 +00:00
29f94d89e1 Settle sequence mode names and identity-based defaults 2026-10-22 14:39:40 +00:00
12f6045da0 Settle sequence combination scope and ordering 2026-10-22 14:08:55 +00:00
6eb80b06e3 Settle sequence identity definitions and policy lifetime 2026-10-22 13:28:18 +00:00
86feb6d755 Settle collection declaration syntax in the sequence roadmap 2026-10-21 14:00:02 +00:00
8bf49f6ea6 Include overlay author controls in the sequence proposal 2026-10-20 14:08:57 +00:00
f3d0fe6411 Record final identity guarantees for application registrations 2026-10-20 14:03:26 +00:00
cba3a0efb0 Record whole-record sequence replacement and application identity constraints 2026-10-20 13:57:19 +00:00
b58cd75c98 Draft sequence identity and layered append proposal 2026-10-20 13:48:54 +00:00
c530a989df Audit operator conventions against Ansible 2026-10-20 13:28:21 +00:00
2baf14d9a5 Retire completed import integration remediation 2026-10-20 04:11:54 +00:00
6903c9e51a Clarify empty path import inspection evidence 2026-10-20 01:06:15 +00:00
31e10af9cb Clarify scheduled provenance and history ordering 2026-10-20 01:05:34 +00:00
0d841d9d73 Resolve contained symlinks in import directory prefixes 2026-10-20 01:04:31 +00:00
4bcefbc9d4 Record import integration audit findings 2026-10-20 00:38:04 +00:00
91db99b967 Retire completed import discovery plans 2026-10-20 00:32:11 +00:00
5747ec49c6 Count import depth only for selected sources 2026-10-20 00:30:35 +00:00
d3a714a1cb Validate complete import depth numbers 2026-10-20 00:27:55 +00:00
7f8ad81065 Track active symlink expansions precisely 2026-10-20 00:26:30 +00:00
75da54150f Resolve native imports beneath nested roots 2026-10-20 00:23:55 +00:00
38fb315415 Check item version ownership at effective paths 2026-10-20 00:22:43 +00:00
1c79ec3aa1 Preserve native import alias identity 2026-10-20 00:20:38 +00:00
01e1b2fc2d Record import discovery review findings 2026-10-20 00:03:23 +00:00
15943e233c Record completed import roadmap 2026-10-19 21:23:54 +00:00
f6a6b0184b Finalize import discovery documentation and audits 2026-10-19 21:23:47 +00:00
898035d83a Record prompt catalog example completion 2026-10-19 21:15:53 +00:00
de3666aed3 Expand prompt catalog discovery example 2026-10-19 21:15:47 +00:00
c00811da2e Record lifecycle audit completion 2026-10-19 21:13:29 +00:00
55abcf8340 Audit scheduled import and inheritance lifecycles 2026-10-19 21:13:21 +00:00
9d7c79cf0b Record public import API completion 2026-10-19 21:06:39 +00:00
24131617c7 Expose import defaults and selection evidence 2026-10-19 21:06:31 +00:00
05b67cbfcf Record empty import evidence completion 2026-10-19 20:57:53 +00:00
e433988962 Retain empty optional import selection evidence 2026-10-19 20:57:41 +00:00
05a28d1178 Record final import wiring completion 2026-10-19 20:51:01 +00:00
1e5bffc0e6 Wire final import declarations through loading 2026-10-19 20:50:47 +00:00
f5a689b578 Record glob discovery completion 2026-10-19 20:38:48 +00:00
e4abff4164 Filter and account final glob candidates 2026-10-19 20:38:42 +00:00
1a8f1cc3a5 Record directory link discovery completion 2026-10-19 20:29:36 +00:00
9b99463187 Follow links safely during directory discovery 2026-10-19 20:29:20 +00:00
f69fa24b40 Record recursive directory search completion 2026-10-19 20:18:44 +00:00
77d97296ea Add recursive directory search with depth limits 2026-10-19 20:18:39 +00:00
032679bf34 Record shallow directory selection completion 2026-10-19 20:13:39 +00:00
55a59de6db Select shallow directory import sources 2026-10-19 20:13:33 +00:00
a0d13962bd Record named import selection completion 2026-10-19 20:08:05 +00:00
5f33e1e90c Select named imports before source admission 2026-10-19 20:07:54 +00:00
b750b0fec6 Record private import parser completion 2026-10-19 20:02:12 +00:00
b498e5bee5 Define private import defaults and declaration parsing 2026-10-19 20:02:05 +00:00
ba577429e6 Record underlay group scheduling completion 2026-10-19 19:53:08 +00:00
014b99504b Schedule underlay groups as dependency forests 2026-10-19 19:53:01 +00:00
263dd2f1bd Record item scope scheduling completion 2026-10-19 19:39:30 +00:00
af43d6f438 Schedule item-local import scopes 2026-10-19 19:39:19 +00:00
a4cbd6390e Record ordinary import scheduling completion 2026-10-19 19:29:52 +00:00
5ed0f55970 Schedule ordinary import contributions by dependency 2026-10-19 19:29:35 +00:00
91ab37a39d Record import graph completion 2026-10-19 17:34:45 +00:00
fbe9ae6d8b Retain import inclusion graph during acquisition 2026-10-19 17:34:21 +00:00
1aa121df71 Record scheduled inheritance completion 2026-10-19 17:25:32 +00:00
6e40c166d6 Schedule sibling inheritance by dependency readiness 2026-10-19 17:25:27 +00:00
a56d4691ff Record inheritance graph groundwork 2026-10-19 15:18:37 +00:00
2718fbb8fe Separate inheritance graph analysis and application 2026-10-19 15:18:25 +00:00
2d52421919 Record deterministic scheduler completion 2026-10-19 15:11:42 +00:00
167fcd475f Add deterministic dependency scheduler 2026-10-19 15:11:28 +00:00
aa93aeda3f Plan import discovery and shared dependency ordering 2026-10-19 14:33:01 +00:00
94381a67fd Record accepted import architecture decisions 2026-10-19 14:24:19 +00:00
c26b34147b Clarify import candidate identity, accounting, and inspection 2026-10-19 14:15:01 +00:00
c9f9c98fbe Settle import validation and require recursion for depth 2026-10-19 14:05:18 +00:00
f6bf4454b0 Settle import selector lists and exclusions 2026-10-19 13:43:56 +00:00
a7ca2c59a2 Settle import defaults, discovery boundaries, and inspection decisions 2026-10-19 01:54:01 +00:00
10c14aa794 Add architecture principles for simplicity and permissive defaults 2026-10-18 21:13:28 +00:00
e9774b252b Adopt shared dependency-ready ordering for imports and inheritance 2026-10-18 21:09:05 +00:00
19fe621df1 Apply permissive defaults and remove import transition support 2026-10-18 16:37:07 +00:00
82b289f4fe Clarify import roadmap decisions using Ansible conventions 2026-10-18 16:28:38 +00:00
160 changed files with 20838 additions and 1698 deletions

View File

@@ -9,7 +9,7 @@ ConfigKit's simple `Load` accepts one strict YAML source (or explicit
defaults-only input). `LoadLayered` additionally combines explicit lower-priority defaults-only input). `LoadLayered` additionally combines explicit lower-priority
source groups with an independently selected main input. Sources can be host source groups with an independently selected main input. Sources can be host
files, native `fs.FS` files, labeled bytes or readers, or snapshotted mappings. files, native `fs.FS` files, labeled bytes or readers, or snapshotted mappings.
Enabled path and glob imports compose strictly within their group; precedence Enabled explicit imports compose strictly within their group; precedence
between groups combines mappings recursively. Both entry points return a complete, between groups combines mappings recursively. Both entry points return a complete,
validated configuration with detached metadata and safe diagnostics. validated configuration with detached metadata and safe diagnostics.
Applications may perform audited defaulting and normalization through a resolver, Applications may perform audited defaulting and normalization through a resolver,
@@ -17,7 +17,8 @@ then report domain errors and warnings through read-only semantic validation. Se
tags and a path predicate drive deterministic redacted effective YAML. Typed secret tags and a path predicate drive deterministic redacted effective YAML. Typed secret
references resolve environment names before validation, and explicit references resolve environment names before validation, and explicit
secrets-directory bootstrap is available as a separate startup operation. Explicit secrets-directory bootstrap is available as a separate startup operation. Explicit
CLI override records apply above overlays. Value explanation is available through CLI override records apply after inheritance and typed decoding, before application
resolution. Value explanation is available through
detached structured records and deterministic redacted output. Built-in detached structured records and deterministic redacted output. Built-in
`extends` relationships let a mapping inherit from a named sibling during the `extends` relationships let a mapping inherit from a named sibling during the
same load, retaining explicit presence and source/derivation history. The root same load, retaining explicit presence and source/derivation history. The root
@@ -26,12 +27,23 @@ the [configuration contract](docs/config.md#effective-configuration-diffs) and
[consumer guide](docs/consumers/README.md#effective-configuration-diffs) describe [consumer guide](docs/consumers/README.md#effective-configuration-diffs) describe
its safe changed-region records and maintained workflows. its safe changed-region records and maintained workflows.
Sources can also be explicitly placed as sequence items, and definitions can Sources can also be explicitly placed as sequence items. Configuration
register selected sequences to append records with distinct string IDs during declarations and application registrations can give selected sequences scalar
strict composition. Higher-priority sequences still replace lower-priority identities and choose how they combine across precedence and inheritance.
sequences as whole values. See the [sequence contract](docs/config.md#strict-document-composition), Strict peers combine distinct records, while higher-priority collections can
[maintained prompt example](docs/consumers/README.md#identified-prompt-sequences), replace, append, or upsert whole records. See the
and [ADR 0014](docs/adr/0014-sequence-placement-and-composition.md). [sequence contract](docs/config.md#strict-document-composition),
[maintained catalog examples](docs/consumers/README.md#identified-prompt-sequences),
and [ADR 0017](docs/adr/0017-collection-sequence-policy.md).
Imports are also explicit: a source may name files, search declared directories,
or select files with path globs. Applications can set load-wide import defaults
while keeping declaration-specific exceptions. Files are acquired in authored
declaration order; dependency-ready composition uses a shared stable scheduler.
ConfigKit does not search conventional application paths implicitly. See the
[import contract](docs/config.md#explicit-imports), [consumer examples](docs/consumers/README.md),
and [ADR 0015](docs/adr/0015-shared-dependency-ready-ordering.md) and
[ADR 0016](docs/adr/0016-independent-import-controls.md).
```go ```go
type Config struct { type Config struct {
@@ -67,12 +79,13 @@ mappings combine, while overlapping non-mapping definitions are errors. Between
precedence tiers, mappings combine recursively, retaining omitted lower-priority precedence tiers, mappings combine recursively, retaining omitted lower-priority
children; other values replace lower-priority values. The main document and its children; other values replace lower-priority values. The main document and its
imports form one strictly composed tier. Explicit underlay groups sit below it imports form one strictly composed tier. Explicit underlay groups sit below it
and above typed defaults; a selected overlay and explicit CLI overrides sit and above typed defaults; a selected overlay applies before inheritance, while
above it. The [configuration contract](docs/config.md#strict-document-composition) explicit CLI overrides apply after decoding and before application resolution.
owns the detailed combination rules and input-specific constraints. At The [configuration contract](docs/config.md#strict-document-composition) owns
definition-registered paths only, strict composition can append records with the detailed combination rules and input-specific constraints. Declared or
distinct explicit IDs; ordinary and unregistered sequences keep their conflict registered collections combine distinct identified records within strict tiers
or whole-value replacement behavior. and use their selected operation at precedence boundaries; other sequences keep
their strict-conflict and whole-value-replacement behavior.
The [architecture goals](docs/policy/architecture.md#success-criteria) explain The [architecture goals](docs/policy/architecture.md#success-criteria) explain
these priorities and how genuine new needs can justify extending the shared these priorities and how genuine new needs can justify extending the shared

View File

@@ -43,12 +43,80 @@ func toEngineLocations(values []Location) []engine.Location {
return result return result
} }
func fromEngineSourceInfo(value engine.SourceInfo) SourceInfo { func fromEngineSourceInfo(value engine.SourceInfo) SourceInfo {
return SourceInfo{ID: value.ID, Label: value.Label, Path: value.Path, Destination: fromEnginePath(value.Destination), PlacementMode: PlacementMode(engine.SourceInfoPlacementModeForFacade(value)), DestinationUnresolved: engine.SourceInfoDestinationUnresolvedForFacade(value), Layer: Layer(value.Layer), Kind: SourceKind(value.Kind), UnderlayIndex: value.UnderlayIndex, ImportedFrom: fromEngineLocationPointer(value.ImportedFrom), VersionCheck: fromEngineSourceVersionCheck(value.VersionCheck)} return SourceInfo{
ID: value.ID, Label: value.Label, Path: value.Path, Destination: fromEnginePath(value.Destination),
PlacementMode: PlacementMode(engine.SourceInfoPlacementModeForFacade(value)),
DestinationUnresolved: engine.SourceInfoDestinationUnresolvedForFacade(value),
Layer: Layer(value.Layer), Kind: SourceKind(value.Kind), UnderlayIndex: value.UnderlayIndex,
ImportedFrom: fromEngineLocationPointer(value.ImportedFrom), VersionCheck: fromEngineSourceVersionCheck(value.VersionCheck),
ImportSelections: fromEngineImportSelections(engine.SourceInfoImportSelectionsForFacade(value)),
}
} }
func toEngineSourceInfo(value SourceInfo) engine.SourceInfo { func toEngineSourceInfo(value SourceInfo) engine.SourceInfo {
result := engine.SourceInfo{ID: value.ID, Label: value.Label, Path: value.Path, Destination: toEnginePath(value.Destination), Layer: engine.Layer(value.Layer), Kind: engine.SourceKind(value.Kind), UnderlayIndex: value.UnderlayIndex, ImportedFrom: toEngineLocationPointer(value.ImportedFrom), VersionCheck: toEngineSourceVersionCheck(value.VersionCheck)} result := engine.SourceInfo{
ID: value.ID, Label: value.Label, Path: value.Path, Destination: toEnginePath(value.Destination),
Layer: engine.Layer(value.Layer), Kind: engine.SourceKind(value.Kind), UnderlayIndex: value.UnderlayIndex,
ImportedFrom: toEngineLocationPointer(value.ImportedFrom), VersionCheck: toEngineSourceVersionCheck(value.VersionCheck),
}
result = engine.SourceInfoWithImportSelections(result, toEngineImportSelections(value.ImportSelections))
return engine.SourceInfoWithPlacement(result, uint8(value.PlacementMode), value.DestinationUnresolved) return engine.SourceInfoWithPlacement(result, uint8(value.PlacementMode), value.DestinationUnresolved)
} }
func fromEngineImportSelections(values []engine.ImportSelectionForFacade) []ImportSelection {
if values == nil {
return nil
}
result := make([]ImportSelection, len(values))
for index, value := range values {
result[index] = ImportSelection{
Path: fromEnginePath(value.Path), Location: fromEngineLocation(value.Location),
Selector: ImportSelector(value.Selector), Outcome: ImportSelectionOutcome(value.Outcome),
}
}
return result
}
func toEngineImportSelections(values []ImportSelection) []engine.ImportSelectionForFacade {
if values == nil {
return nil
}
result := make([]engine.ImportSelectionForFacade, len(values))
for index, value := range values {
result[index] = engine.ImportSelectionForFacade{
Path: toEnginePath(value.Path), Location: toEngineLocation(value.Location),
Selector: string(value.Selector), Outcome: string(value.Outcome),
}
}
return result
}
func toEngineImportDefaults(value ImportDefaults) engine.ImportDefaults {
return engine.ImportDefaults{
Required: cloneImportDefaultBool(value.Required),
Follow: cloneImportDefaultBool(value.Follow),
Hidden: cloneImportDefaultBool(value.Hidden),
Recursive: cloneImportDefaultBool(value.Recursive),
Pattern: cloneImportDefaultStrings(value.Pattern),
}
}
func cloneImportDefaultBool(value *bool) *bool {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneImportDefaultStrings(values []string) []string {
if values == nil {
return nil
}
cloned := make([]string, len(values))
copy(cloned, values)
return cloned
}
func fromEngineSourceVersionCheck(value *engine.SourceVersionCheck) *SourceVersionCheck { func fromEngineSourceVersionCheck(value *engine.SourceVersionCheck) *SourceVersionCheck {
if value == nil { if value == nil {
return nil return nil

View File

@@ -1,6 +1,8 @@
# ADR 0005: Record explicit CLI overrides and safe value explanation # ADR 0005: Record explicit CLI overrides and safe value explanation
Status: accepted Status: Partially superseded by
[ADR 0018](0018-cli-after-structural-resolution.md) for override-before-decoding
ordering. The remaining ownership and safe explanation decisions remain accepted.
Date: 2026-09-06 Date: 2026-09-06

View File

@@ -2,7 +2,9 @@
## Status ## Status
Proposed draft for review. Completed planning and remediation records are Proposed draft for review; its whole-sequence precedence replacement rule is
superseded by [ADR 0017](0017-collection-sequence-policy.md). The remainder
retains its proposed status. Completed planning and remediation records are
preserved in Git history at `b604aa1` under `docs/roadmap/`; the preserved in Git history at `b604aa1` under `docs/roadmap/`; the
[configuration contract](../config.md#layered-loading) owns current behavior. [configuration contract](../config.md#layered-loading) owns current behavior.

View File

@@ -2,8 +2,14 @@
## Status ## Status
Accepted and implemented. The [configuration contract](../config.md#sibling-inheritance) Partially superseded by [ADR 0017](0017-collection-sequence-policy.md) for
owns current behavior. The completed implementation plan is preserved in Git sequence combination and the addition of identified-sequence sibling scopes,
and [ADR 0018](0018-cli-after-structural-resolution.md) for CLI-before-inheritance
ordering. Existing mapping-sibling scopes and the remaining ownership and
inspection decisions remain accepted.
The [configuration contract](../config.md#sibling-inheritance) owns implemented
behavior. The completed implementation plan is preserved in Git
history at `83c59ea`. history at `83c59ea`.
## Date ## Date

View File

@@ -2,8 +2,12 @@
## Status ## Status
Accepted and implemented. The [configuration contract](../config.md#strict-document-composition) Partially superseded by [ADR 0017](0017-collection-sequence-policy.md) for
defines current behavior, and the [consumer guide](../consumers/README.md#identified-prompt-sequences) identity ownership and sequence combination. Explicit source placement remains
accepted.
The [configuration contract](../config.md#strict-document-composition)
defines implemented behavior, and the [consumer guide](../consumers/README.md#identified-prompt-sequences)
shows the maintained example. The completed roadmap, implementation plan, and shows the maintained example. The completed roadmap, implementation plan, and
resolved remediation record are preserved in Git history at `008d2ba` under resolved remediation record are preserved in Git history at `008d2ba` under
`docs/roadmap/`; see the [development guide](../development.md) for retrieval `docs/roadmap/`; see the [development guide](../development.md) for retrieval

View File

@@ -0,0 +1,68 @@
# ADR 0015: Share Dependency-Ready Ordering Across Imports and Inheritance
## Status
Accepted. Implemented behavior is described in the
[configuration contract](../config.md#explicit-imports) and
[internal import overview](../internal/imports.md).
## Date
2026-10-19
## Context
Imports and sibling inheritance both resolve relationships between configuration
contributions. Their existing resolvers use different traversal rules, making
ordering depend on which mechanism expresses a dependency. Ordering affects
sequence contributions, provenance, and diagnostics, so a consistent rule is
part of ConfigKit's configuration model.
The resolvers also serve distinct purposes. Imports strictly compose sources;
inheritance overlays a child on its resolved parent after source assembly.
Item-local contributions and nested inheritance introduce scope boundaries that
resolution must preserve.
## Decision
ConfigKit owns one shared dependency-ready scheduler for imports and sibling
inheritance. Both resolvers use Kahn's algorithm with a stable ordering key,
selecting the smallest currently eligible node after its dependencies have
completed. The shared scheduler owns eligibility, deterministic tie-breaking,
and dependency-cycle identification. Each resolver owns construction of its
dependency relationships and application of its domain operations.
Apply the common scheduling rule within the scopes where dependencies are
available. Complete item-local contributions before item placement and resolve
enclosing inheritance before relationships in the resulting nested scopes.
Source discovery supplies the import graph; the scheduler determines resolution
order.
Strict composition, inheritance overlay, and precedence tiers retain their
existing ownership and meanings under [ADR 0001](0001-separate-composition-and-overlay.md).
The lifecycle and scope decisions in
[ADR 0009](0009-sibling-inheritance.md) continue to govern inheritance. Import
admission retains its cycle and repeated-source rules.
## Alternatives Considered
- **Deterministic depth-first postorder.** Keeps dependency branches together,
but gives traversal branch selection an additional role in ordering.
- **Dependency rounds.** Groups nodes by readiness rounds, but delays newly
eligible nodes until the current round finishes.
- **Descending longest-path depth.** Prioritizes the furthest dependencies,
but adds distance from the root as an ordering policy.
- **Separate resolver algorithms.** Allows local specialization, but leaves
consumers and maintainers with different rules for analogous dependencies.
## Consequences
Both resolvers follow one deterministic ordering model and share responsibility
for maintaining that model. Resolver-specific scopes, combination semantics,
and source-admission rules remain explicit at their integration boundaries.
Independent branches may interleave as nodes become eligible. Adopting this
rule changes existing traversal-dependent sequence positions and reporting
order, requiring coordinated review of source assembly, provenance, and
inheritance behavior. The configuration contract owns the implemented ordering
details.

View File

@@ -0,0 +1,83 @@
# ADR 0016: Separate Import Selection, Traversal, and Presence Requirements
## Status
Accepted. Implemented behavior is described in the
[configuration contract](../config.md#explicit-imports), the
[public API declarations](../../lifecycle.go), and the
[internal import overview](../internal/imports.md).
## Date
2026-10-19
## Context
Consumers assemble configurations from named files and collections whose layout
varies between installations. Simple imports should require little setup, while
larger collections need control over discovery, recursion, filtering, and
required contributions. Combining those choices implicitly makes declarations
harder to interpret and pushes source-selection work into applications.
Discovery can also reach a candidate through overlapping searches or encounter
different names for one canonical source. Selection and source admission need
distinct identities to support useful search unions while preserving ConfigKit's
existing source-integrity rules.
## Decision
ConfigKit owns import discovery and selection within its source-loading
lifecycle. Support explicit files, directory searches, and path globs as
distinct selection forms. Keep traversal, filename filtering, and presence
requirements independently expressible. Discovery determines the selected
contributions; the shared scheduler in
[ADR 0015](0015-shared-dependency-ready-ordering.md) determines their resolution
order.
Use optional-by-default imports with explicit requiredness controls. Selected
sources remain subject to ConfigKit's validity, containment, resource, and
composition contracts. This applies the library's
[simplicity and defaults principles](../policy/architecture.md#simplicity-defaults-and-explicit-controls)
to import absence while retaining meaningful errors for invalid contributions.
Applications own their selected import defaults. Authored declaration values
take precedence over application defaults, which take precedence over library
defaults. Each declaration resolves applicable controls independently,
including declarations in imported sources. This gives applications a shared
baseline while preserving explicit choices in individual declarations.
Keep declaration-local candidate identity separate from canonical source
identity. Search unions coalesce repeated candidate paths within a source
namespace before symlink resolution; filename filtering uses the selected or
discovered name. Source admission continues to use canonical identity and the
existing repeated-source rules. Distinct aliases retain their separate
inclusion routes.
## Alternatives Considered
- **Use path globs for all discovery.** Offers one selection form, but makes
collection traversal depend on path-pattern structure and complicates
independently controlling recursion and filename filtering.
- **Offer only explicit files and directory searches.** Reduces the selector
vocabulary, but moves wildcard directory selection into repeated declarations
or application-side discovery.
- **Require every import by default.** Detects absent contributions immediately,
but requires repeated opt-outs for installations with optional components.
- **Make application settings override authored declarations.** Provides central
enforcement, but prevents declarations from expressing local exceptions.
- **Deduplicate selections globally by canonical source identity.** Coalesces
aliases and separate inclusion routes, but changes the existing repeated-source
contract and obscures independently authored contributions.
## Consequences
Consumers can express simple imports and more selective collection searches
through the same loading lifecycle. Applications choose defaults and validate
their required effective configuration; ConfigKit owns consistent selection,
source admission, composition, and inspection.
Optional absence can conceal misspelled selectors, so requiredness and
inspection provide explicit ways to detect and explain missing contributions.
Declaration-local deduplication requires the loader to preserve candidate
identity alongside canonical source identity. The configuration and public API
contracts own the implemented behavior.

View File

@@ -0,0 +1,143 @@
# ADR 0017: Define Identity and Merge Rules for Each Collection
## Status
Accepted.
This supersedes the decisions about who declares identity and how
sequences combine in [ADR 0014](0014-sequence-placement-and-composition.md),
and the rule that inheritance always replaces sequences in
[ADR 0009](0009-sibling-inheritance.md). It also extends that ADR's sibling
scopes to identified sequences. Existing mapping-sibling scopes and source
placement continue to apply. It supersedes the whole-sequence
precedence replacement rule in the proposed
[ADR 0007](0007-source-backed-underlays.md).
## Date
2026-10-22
## Context
ConfigKit can load separate files as records in a sequence. Applications can
register an identity key so records with different identities combine and
duplicates fail. However, a higher configuration layer or an inheriting mapping
currently replaces the whole sequence. Applications that need to extend a
catalog across layers must write their own merging code and keep source and
history information consistent with its results.
Configuration authors need to choose how records are identified and whether a
layer adds records, updates records, or replaces the collection. Applications
also need a way to require unique identities in the final loaded configuration.
These rules must work consistently for imported records, inline sequences,
overlays, and inheritance.
Records in a sequence may also share settings. Authors need to reuse those
settings through inheritance while keeping each record's own identity.
## Decision
ConfigKit manages identity and merge rules for each collection during loading.
Configuration files can declare those rules for imported and inline sequences,
including sequences in selected overlays. Applications can require a particular
identity definition, and configuration declarations must agree with it. Source
placement determines where each contribution enters the configuration.
Keep the distinction between contributions at the same priority and overrides
from a higher layer, as established in
[ADR 0001](0001-separate-composition-and-overlay.md). Within one tier, records
with distinct identities combine; duplicate identities and conflicting
declarations fail. Determine the applicable rules before combining those
contributions, including any identity established in lower tiers.
Across layers, support three operations: replace the whole sequence, append
records while rejecting duplicate identities, or replace whole records by
identity (`upsert`). Source layering and sibling inheritance use the same
operations. For inheritance, the rules at the inheriting destination determine
which operation applies.
An identified sequence is also a sibling scope for inheritance. A record may
inherit from one other record in the same sequence, selected by its complete
identity. Each record supplies its own identity before inheritance. The child
keeps that identity and combines its settings with the parent's other fields,
using the existing mapping and collection rules. Parents remain ordinary records
subject to normal validation.
Resolve these relationships after source assembly, using the same dependency
ordering as mapping-sibling inheritance. Resolve nested relationships at each
inheriting destination. Preserve the assembled sequence order. Whole-record
upsert keeps its existing meaning; inheritance then uses the records that remain.
Typed defaults, CLI overrides, and application resolution follow the lifecycle
in [ADR 0018](0018-cli-after-structural-resolution.md).
An explicitly selected operation remains in effect at its destination until a
higher layer changes it. When no explicit operation applies, use upsert if an
identity definition is available; otherwise replace the whole sequence. Authors
can declare identity and an operation before supplying any records. Keep source
acquisition order separate from layer priority and the dependency ordering
established by [ADR 0015](0015-shared-dependency-ready-ordering.md).
Rules declared inside a record belong to that record. Retaining the record
retains its local rules; replacing it discards them. Replacing a whole collection
can discard its previous YAML identity definition, while application requirements
still apply. Inheritance can copy collection identity, but the inheriting
destination chooses its operation. Application registrations continue to govern
the paths where they were registered. The same ownership rules apply to
declarations made before a sequence exists.
Check YAML-declared identity when combining collections during source assembly
and inheritance. Application registrations also require the final loaded
sequences to satisfy their identity rules, including after typed defaults, CLI
overrides, and application resolution. This guarantee checks sequences that
exist; the application schema and validation decide whether a collection must
exist and which value types are allowed. The guarantee applies when loading
completes, rather than to every intermediate callback state or changes made by
the caller afterward.
Preserve explicit presence, source attribution, history, sensitivity, and
resource context through these operations, following
[ADR 0003](0003-preserve-presence-and-resolution-history.md). Typed defaults,
CLI assignments, and application resolution keep their own rules for changing
values. The public API provides registration and inspection; the implementation
chooses how to store and track collection rules internally.
## Alternatives Considered
- **Keep identity application-owned and always replace sequences across layers.**
Keeps the current design small, but reusable catalogs require application
changes or custom merging code.
- **Put merge rules on each import.** Keeps the rules beside the imported source,
but inline sequences and overlays need a separate way to declare them. It also
makes rules for a collection spread across several imports harder to reconcile.
- **Merge individual fields of records with matching identities.** Allows partial
record updates, but adds rules for field conflicts and retained values.
Replacing the whole record makes the update's scope clear.
- **Keep inheritance limited to mapping siblings.** Keeps parent lookup simpler,
but authors of sequence records must duplicate shared settings or applications
must implement inheritance themselves. Collection identity provides a way to
select a parent within the existing loading process.
- **Check every YAML identity again at the end of loading.** Gives YAML declarations
authority over values changed by defaults, CLI assignments, and application
resolution. This extends the lifetime of those declarations; application
registration provides that final guarantee when needed.
## Consequences
Consumers can build and customize catalogs in one load while retaining useful
inspection information. Configuration authors choose how collections combine
within application requirements. Duplicate identities among contributions at
the same priority remain errors.
Existing registered collections use upsert across layers by default unless an
explicit operation selects otherwise. Correct loading now requires tracking
which declarations still apply as records are replaced, copied, or moved to new
positions. Lower-tier identity must be available before combining higher-tier
contributions.
Record inheritance also requires parent lookup by identity and checking for
missing parents and cycles. Copied settings retain the same source, history,
and sensitivity information as mapping-sibling inheritance.
The [configuration contract](../config.md#strict-document-composition) defines
declaration syntax, identity comparison, omission and empty-value behavior, and
diagnostics for the implemented feature.

View File

@@ -0,0 +1,84 @@
# ADR 0018: Apply CLI Overrides After Inheritance and Decoding
## Status
Accepted.
This supersedes the rule that CLI overrides apply before decoding in
[ADR 0005](0005-explicit-overrides-and-safe-explanation.md) and the
rule that they apply before inheritance in [ADR 0009](0009-sibling-inheritance.md).
Their remaining ownership, inheritance scope, and safe inspection decisions
continue to apply.
## Date
2026-10-22
## Context
Explicit command-line values are the operator's highest-priority input. If they
apply before inheritance, a sequence supplied as a replacement could gain
inherited records afterward. Its final contents would then differ from the
operator's assignment.
The application also needs these values when it fills missing settings,
normalizes values, or computes derived settings. ConfigKit needs a clear loading
order that preserves both CLI precedence and the application's role in resolving
the final configuration.
## Decision
ConfigKit uses this loading order:
1. Combine sources within tiers and apply precedence between layers.
2. Resolve sibling inheritance, apply typed defaults, and decode the configuration
into the application's types.
3. Apply explicit CLI overrides and check them against the application schema.
4. Run application resolution using the overridden configuration.
5. Look up effective secret references, validate the final configuration, and
return it.
CLI overrides are the highest-priority input to application resolution. A CLI
sequence replaces the whole sequence; mappings combine recursively. Collection
merge rules apply during the earlier source and inheritance steps. Inheritance
and typed defaulting do not run again after CLI overrides.
Keep track of which values were explicitly supplied and which CLI input supplied
them. Applications continue to parse command-line arguments; ConfigKit applies
and validates the overrides and makes their effects inspectable. Reject
structural directives in override paths and values, so instructions such as
inheritance relationships remain part of source processing.
Application resolution keeps its existing rules for defaults and recorded
changes under [ADR 0003](0003-preserve-presence-and-resolution-history.md).
It receives CLI values together with their presence and source information.
Defaults that only fill missing settings preserve explicit CLI values.
Application normalization and derived-value calculations can still change them,
with those changes recorded. Secret lookup and final validation use the resulting
configuration.
## Alternatives Considered
- **Apply CLI overrides before inheritance.** Preserves existing ordering, but
later merging needs special handling to preserve CLI replacement intent while
still combining ordinary source contributions.
- **Apply CLI overrides after inheritance but before typed decoding.** Allows an
override to fix a source value that would otherwise fail decoding, but leaves
typed defaulting after the operator's assignment. Completing defaulting and
decoding first gives CLI overrides an already resolved configuration to modify.
- **Apply CLI overrides after application resolution.** Applies assignments after
the application's changes, but derived settings still reflect earlier values
unless resolution runs again.
## Consequences
CLI overrides change a configuration whose inheritance and typed defaults are
already resolved. The application can then derive settings from the operator's
values. The final values, source information, history, and sensitivity handling
must remain consistent throughout loading.
CLI input can no longer add or cancel inheritance relationships or bypass an
earlier error in source assembly or inheritance. It also cannot fix a value that
fails the earlier decoding step. Semantic validation still runs after overrides
and application resolution. The [configuration contract](../config.md#explicit-cli-overrides)
and [consumer guidance](../consumers/README.md) describe this behavior.

View File

@@ -52,10 +52,11 @@ preserves already validated secret state without rereading the environment.
The current lifecycle is: The current lifecycle is:
```text ```text
validate options → parse the selected source → traverse/compose enabled imports validate options → parse the selected source → extract source controls and
traverse/strictly compose enabled imports
→ build schema → check the optional application version → apply a selected overlay → build schema → check the optional application version → apply a selected overlay
→ apply explicit CLI overrides → expand sibling inheritance → expand sibling inheritance
→ create and clone defaults → strictly decode → create and clone defaults → strictly decode → apply explicit CLI overrides
→ recorded application resolution → recorded application resolution
→ resolve typed secret references → semantic validation → classify secrets → resolve typed secret references → semantic validation → classify secrets
→ snapshot configuration and metadata → snapshot configuration and metadata
@@ -68,17 +69,18 @@ contributes no source, and continues through defaults, decode, and projection. A
missing file remains an error; it is never interpreted as no input. missing file remains an error; it is never interpreted as no input.
`Options` accepts the application definition, ordinary-source `Destination` and `Options` accepts the application definition, ordinary-source `Destination` and
`PlacementMode`, explicit path and glob imports, one selected overlay, filesystem `PlacementMode`, explicit imports, one selected overlay, filesystem
context, explicit CLI overrides, and shared source-version controls. The definition context, explicit CLI overrides, and shared source-version controls. The definition
may register strict sequence composition paths through may register sequence identities through `Definition.SequenceIdentities`. Its
`SequenceCompositions`. Its zero-valued placement mode places the source mapping zero-valued placement mode places the source mapping
at the destination; `sequence-item` explicitly wraps the mapping as one item at a at the destination; `sequence-item` explicitly wraps the mapping as one item at a
non-root destination. Unknown modes, invalid destinations, and invalid or non-root destination. Unknown modes, invalid destinations, and invalid or
conflicting sequence registrations fail before source acquisition. Overlay and conflicting sequence registrations fail before source acquisition. Overlay and
CLI coordinates remain rooted at the application root. `NoSource` rejects CLI coordinates remain rooted at the application root. Ordinary `Load` recognizes
source-root collection declarations whether or not imports are enabled; an
`imports` declaration still requires imports to be enabled. `NoSource` rejects
imports, active overlays, and filesystem context as invalid combinations but imports, active overlays, and filesystem context as invalid combinations but
accepts overrides. Input containing a reserved `configkit` mapping key is rejected accepts overrides and remains an absent source rather than an empty document.
while imports are disabled.
Every source, options, schema, version, defaults, decode, resolution, or validation Every source, options, schema, version, defaults, decode, resolution, or validation
failure returns a nil result and a `*LoadError`. Successful results contain caller-owned `Config`, detached failure returns a nil result and a `*LoadError`. Successful results contain caller-owned `Config`, detached
@@ -128,16 +130,20 @@ nothing. Underlay groups are listed from lowest to highest priority. Repeating
a file within one group remains an import/repetition error; the same reusable a file within one group remains an import/repetition error; the same reusable
source may be selected again in another tier. source may be selected again in another tier.
ConfigKit acquires and composes the main input first, constructs the schema, ConfigKit acquires the main input first and constructs the schema. It then
and checks the optional application version from the main tier only. Underlay acquires and strictly composes underlay groups from lowest to highest priority,
roots cannot claim that version path (`underlay.version`); imported fragments passing each group's accumulated lower collection policy into the next group.
retain `import.version` protection. It then acquires and composes underlay Each group is applied once as a precedence tier. The main tier is then composed
groups in order, recursively combines the groups, and applies the main tier, with the accumulated underlay policy, and the optional application version is
one selected overlay, and explicit CLI overrides. Scalar, sequence, and null checked against that composed main tier only. Underlay roots cannot claim that
definitions replace lower values between tiers; mappings combine recursively. version path (`underlay.version`); imported fragments retain `import.version`
Syntax and strict-composition errors inside a tier cannot be masked by a protection. ConfigKit then applies the selected overlay and resolves inheritance.
higher-priority tier. Defaults, typed decoding, resolution, secret lookup, Scalar, sequence, and null definitions replace lower values between tiers unless
validation, and publication run once on the final assembled input. a collection policy defines the sequence operation; mappings combine recursively.
After successful defaults and typed decoding, CLI overrides are applied to that
typed configuration before application resolution. Syntax, strict-composition,
inheritance, and decoding errors cannot be masked by a CLI override. Resolution,
secret lookup, validation, and publication run once on the resulting values.
An active layered overlay requires `LayeredOptions.OverlayFilesystem` explicitly, An active layered overlay requires `LayeredOptions.OverlayFilesystem` explicitly,
independent of the main source's namespace. Its name and root-relative lookup independent of the main source's namespace. Its name and root-relative lookup
@@ -147,7 +153,10 @@ Ordinary `Load` continues to infer its selected-overlay context from its source.
Source IDs follow acquisition order—main and its depth-first imports, then each Source IDs follow acquisition order—main and its depth-first imports, then each
underlay tier's inputs and depth-first imports, then overlay and CLI—even though underlay tier's inputs and depth-first imports, then overlay and CLI—even though
history follows precedence order. Source `Layer` describes precedence while history follows precedence order. Default main-version checking happens after
underlay composition because the main tier inherits the accumulated collection
policy. Enabled per-source version checks retain acquisition order and source
local headers. Source `Layer` describes precedence while
`Kind` describes acquisition. Underlay records use `LayerUnderlay` and a `Kind` describes acquisition. Underlay records use `LayerUnderlay` and a
one-based `UnderlayIndex` giving their original tier position; all other records one-based `UnderlayIndex` giving their original tier position; all other records
have index zero. Explicit empty groups and `NoSource` consume no source ID. have index zero. Explicit empty groups and `NoSource` consume no source ID.
@@ -213,28 +222,55 @@ is recorded in [ADR 0011](adr/0011-input-destinations.md) and
Any input mapping may declare `extends: name` to inherit from the mapping named Any input mapping may declare `extends: name` to inherit from the mapping named
`name` among its immediate siblings. Names are exact keys, including the empty `name` among its immediate siblings. Names are exact keys, including the empty
string; lookup does not search ancestor or unrelated scopes. The document root string; lookup does not search ancestor or unrelated scopes. A mapping inside a
and a sequence-element mapping have no sibling namespace for their own string sequence can also participate in record inheritance when that sequence has an
relationship, though mappings nested inside them can have one. An `extends` effective identity declaration from `configkit.sequences` or an application
registration. In that case, `extends` is a mapping containing exactly the
sequence's identity fields, with scalar values; lookup is limited to that final
sequence. The document root and a sequence-element mapping without an effective
identity declaration have no sibling namespace for non-null string
relationships, though mappings nested inside them can have one. An `extends`
value of null cancels a relationship without looking up a parent. Other value value of null cancels a relationship without looking up a parent. Other value
types, missing or non-mapping parents, and cycles fail the load with value-free types, incomplete or extra reference fields, missing parents, and cycles fail
diagnostics. the load with value-free diagnostics.
Within each mapping sibling scope or identified sequence, direct definitions
resolve by dependency readiness. A parent is resolved before a child; among
currently eligible definitions, the lexically smallest canonical semantic
pointer is selected, including newly eligible definitions as soon as their
parent completes. This ordering governs inheritance operations and their
history, controls, and transfers. It does not reorder mappings or sequence
records in the resulting configuration.
`extends` is reserved structural input across ordinary and layered loads, every `extends` is reserved structural input across ordinary and layered loads, every
source kind, imports, selected overlays, and explicit CLI overrides. It is source kind, imports, selected overlays, and explicit CLI overrides. It is
consumed before typed decoding and need not appear in the application's Go consumed before typed decoding and need not appear in the application's Go
types. Applications previously using `extends` as an ordinary YAML field must types. Applications previously using `extends` as an ordinary YAML field must
rename that key and its field. No collection registration or keyword setup is rename that key and its field. No collection registration or keyword setup is
required. required for mapping-sibling relationships; identified-sequence relationships
require effective identity fields for that collection.
ConfigKit completes strict composition within tiers and precedence assembly ConfigKit completes strict composition within tiers and precedence assembly
between tiers before resolving relationships. An omitted `extends` retains a between tiers before resolving relationships. Only records that survive that
lower-priority relationship; `extends: null` cancels it while preserving the assembly participate in identified-sequence lookup: an upserted parent supplies
child's own settings. Superseded references are historical evidence, not active its final record, a replaced parent is unavailable, and a higher replacement
relationships to validate. Each child recursively overlays its own settings without `extends` supersedes a lower relationship. An omitted `extends` retains
over its resolved parent: explicit zero, false, empty string, sequence, and a lower-priority mapping-sibling relationship; an identified record's effective
null values replace inherited values, while mappings combine recursively. directive depends on the record that survives assembly. `extends: null` cancels
The child retains its key. All effective relationships resolve, including a relationship while preserving the child's own settings. Superseded references
are historical evidence, not active relationships to validate. Each child
recursively overlays its own settings over its resolved parent: explicit zero,
false, empty string, and
null values replace inherited values, while mappings combine recursively. For
identified records, the child's identity fields and their origins remain its
own; parent identity fields do not contribute to the payload.
Sequences at collection destinations use the shared collection operation with
the parent as lower input and the child as higher input. The destination's
explicit mode applies; otherwise a copied identity selects upsert and an
unkeyed destination replaces. A parent's explicit mode does not move to a
different destination. Sequences outside collection destinations remain atomic
replacements.
All effective relationships resolve, including
entries the application may never select for execution. Partial parent entries the application may never select for execution. Partial parent
definitions undergo normal typed decoding and application-owned validation, definitions undergo normal typed decoding and application-owned validation,
not a special structural exemption. not a special structural exemption.
@@ -416,20 +452,22 @@ the descendant can therefore retain history after `Lookup` stops reporting it as
effective. Equal-valued input is still an explicit base definition, while a derived effective. Equal-valued input is still an explicit base definition, while a derived
value for absent input remains non-explicit. value for absent input remains non-explicit.
Appending registered records during strict composition assigns each contribution Strict composition assigns contributed records their assembled sequence positions
its assembled sequence index in history. A later precedence tier that supplies a in history. Across tiers, history follows the selected
sequence records replacement at the sequence path; history does not imply [collection operation](#strict-document-composition), including append and
identity-based merging across tiers. whole-record upsert. Incoming contributions use their assigned positions, and
replaced payload remains available as historical evidence.
A mapping assembled from the selected file and imports has one composite base origin A mapping assembled from the selected file and imports has one composite base origin
whose locations retain source traversal order. A mapping also visited by an overlay whose locations follow [scoped assembly order](#explicit-imports), not source-record
has an overlay-layer composite origin with the lower contributing locations followed acquisition order. A mapping also visited by an overlay has an overlay-layer
by the overlay location; an empty mapping is still an explicit container even though composite origin with the lower contributing locations followed by the overlay
it contributes no children. Resolution origins are synthetic: they have layer location; an empty mapping is still an explicit container even though it contributes
no children. Resolution origins are synthetic: they have layer
`resolution`, no fabricated file location, and a `derived_default` or `normalization` `resolution`, no fabricated file location, and a `derived_default` or `normalization`
operation. Reasons and dependency paths retain first occurrence order with duplicates operation. Reasons and dependency paths retain first occurrence order with duplicates
removed. Source records remain ordered as selected source, depth-first imports, the removed. Source records and IDs retain [acquisition order](#explicit-imports),
selected overlay, then synthetic CLI records in supplied order. including the underlay ordering described in [layered loading](#layered-loading).
Inheritance adds no acquired source or precedence layer. `Origin.Inheritance` Inheritance adds no acquired source or precedence layer. `Origin.Inheritance`
lists concrete transfers from contributor paths through destinations, with lists concrete transfers from contributor paths through destinations, with
@@ -492,29 +530,34 @@ historical path. Its effective value and each chronological before/after value a
`ValueSnapshot` records containing deterministic YAML text rather than mutable Go or `ValueSnapshot` records containing deterministic YAML text rather than mutable Go or
parser values. `Present=false` with empty YAML means absence or deletion; explicit parser values. `Present=false` with empty YAML means absence or deletion; explicit
null is present and renders as `null`. The history starts with the canonical typed null is present and renders as `null`. The history starts with the canonical typed
application-default or Go-zero baseline, then includes only relevant base sources in application-default or Go-zero baseline, followed by relevant underlay and main-tier
traversal order, the selected overlay, supplied CLI records, and audited net contributions in precedence order, the selected overlay, and structural inheritance
resolution operations. Structural inheritance events appear after source replay events. Typed defaulting and decoding reconcile that source history before supplied
and before typed reconciliation, including consumed or canceled `extends` input CLI records and audited net resolution operations. Within each strict tier, replay follows
and equal-valued child replacements. Overlapping parent/child resolution records [scoped assembly order](#explicit-imports), not source-record acquisition order.
Inheritance events include consumed or canceled `extends` input and equal-valued
child replacements. Overlapping parent/child resolution records
are rendered are rendered
as one net subtree event with combined reasons and dependencies, rather than as one net subtree event with combined reasons and dependencies, rather than
pretending that the recorded assignments ran in lexical order. Explicit equal pretending that the recorded assignments ran in lexical order. Explicit equal
writes remain visible. When input replay writes remain visible. When structural replay differs from the pre-CLI typed input,
differs from the final typed input, a `ChangeDecoding` event records the transition a `ChangeDecoding` event records retained defaults and canonical decoding. CLI
before domain resolution. Its origin retains the decoded path's source/default contributions then retain their raw input representation while effective and
attribution; its reason identifies static-default retention and canonical decoding. resolution values use the typed projection. This built-in inspection operation is
This built-in inspection operation is not accepted by `Resolution.Record`. not accepted by `Resolution.Record`.
Base, overlay, and CLI history preserves the policy-validated input representation, Base, overlay, and CLI history preserves the policy-validated input representation,
including a superseded value that could not have decoded into the final Go field. including a superseded value that could not have decoded into the final Go field.
Baseline, effective, and resolution values use canonical typed projection. A Baseline, effective, and resolution values use canonical typed projection. A
decoding event bridges replayed input and canonical typed values, so intermediate decoding event bridges replayed input and canonical typed values, so intermediate
null/scalar replacements cannot incorrectly imply permanent default removal. Mapping null/scalar replacements cannot incorrectly imply permanent default removal. Mapping
queries reconstruct each source or override contribution, while sequences remain queries reconstruct each source or override contribution. A sequence snapshot
atomic: a descendant history identifies its presence before and after the containing represents the collection at that point in replay, and sequence diffs remain
sequence replacement without implying element-wise merging. Removed paths remain atomic. Descendant queries describe their presence and values before and after
explainable; unknown and malformed paths return false. containing sequence operations. Combination follows the
[collection operation tables](#strict-document-composition); upsert replaces
matching records in full. Removed paths remain explainable; unknown and malformed
paths return false.
Every snapshot applies the same historical tag, predicate, and recorded-dependency Every snapshot applies the same historical tag, predicate, and recorded-dependency
sensitivity closure as effective rendering before YAML is exposed. A redacted scalar sensitivity closure as effective rendering before YAML is exposed. A redacted scalar
@@ -601,13 +644,15 @@ must resolve to a nonempty value. Secret references work in typed structs, maps,
pointers, slices, and arrays. A dynamic `any` value does not establish secret pointers, slices, and arrays. A dynamic `any` value does not establish secret
semantics and cannot contain `Secret`; applications must use a typed slot. semantics and cannot contain `Secret`; applications must use a typed slot.
ConfigKit resolves only the effective names after recorded application resolution ConfigKit resolves only the effective names after CLI assignment and recorded
and before semantic validation. Each unique name is read with `os.LookupEnv` once application resolution, and before semantic validation. CLI overrides may supply
per load. Validation can inspect the credential through the deliberately named or replace a typed secret reference; lookup uses that effective name and retains
`Value()` accessor and distinguish success with `IsResolved()`. Later loads read its effective source location for diagnostics. Each unique name is read with
the environment again, and cloned defaults are rebuilt from names so a previously `os.LookupEnv` once per load. Validation can inspect the credential through the
resolved result cannot retain a stale credential. `Load` never writes to the deliberately named `Value()` accessor and distinguish success with `IsResolved()`.
environment. Later loads read the environment again, and cloned defaults are rebuilt from names
so a previously resolved result cannot retain a stale credential. `Load` never
writes to the environment.
Credential bytes are held only in the per-load lookup state and the trusted returned Credential bytes are held only in the per-load lookup state and the trusted returned
configuration, and are returned only by `Value`. Secret formatting uses a safe configuration, and are returned only by `Value`. Secret formatting uses a safe
@@ -671,18 +716,19 @@ files when appropriate; ConfigKit stores no implicit global report.
### Recorded application resolution ### Recorded application resolution
An optional `Definition.Resolve` callback runs after successful typed decoding. It An optional `Definition.Resolve` callback runs after successful defaults, typed
receives the working configuration and a per-load `Resolution` context. The context's decoding, and explicit CLI assignment. It receives the working configuration and a
per-load `Resolution` context. The context's
`IsExplicit` query uses surviving input presence, so explicit false, zero, empty, and `IsExplicit` query uses surviving input presence, so explicit false, zero, empty, and
null are never confused with absence or static defaults. null are never confused with absence or static defaults.
`Resolution.InputInfo(path)` returns detached effective metadata from the fixed `Resolution.InputInfo(path)` returns detached effective metadata from the fixed
decoded entry state, before application resolution runs. It includes retained static post-decode entry state, including explicit CLI overrides, before application
defaults, preserves explicit null, and uses the same exact semantic path matching as resolution runs. It includes retained static defaults, preserves explicit null, and
`Metadata.Lookup`; missing, malformed, and key/index-shape-mismatched paths return uses the same exact semantic path matching as `Metadata.Lookup`; missing,
false without diagnostics. Its `Explicit` value agrees with `IsExplicit` for every malformed, and key/index-shape-mismatched paths return false without diagnostics.
found entry path. Resolution changes, additions, removals, and moves do not alter Its `Explicit` value agrees with `IsExplicit` for every found entry path. Resolution
this entry view. changes, additions, removals, and moves do not alter this entry view.
`Resolution.Source(id)` returns a detached record for an already acquired source in `Resolution.Source(id)` returns a detached record for an already acquired source in
the same load: the main source, imports, selected overlay, and supplied CLI records the same load: the main source, imports, selected overlay, and supplied CLI records
@@ -757,8 +803,11 @@ live services as a substitute for configuration validation.
## Strict document composition ## Strict document composition
ConfigKit strictly composes parsed base documents when `Options.Imports` is true. Ordinary `Load` applies strict document composition to its parsed source scope;
Application defaults do not participate in this operation. `Options.Imports` adds imported documents as peers in that scope. Application
defaults do not participate. With imports disabled, the single source still has
its controls extracted and any declared or registered collection identity is
validated, but there are no imported peers to combine.
| Existing node | Incoming node | Result | | Existing node | Incoming node | Result |
| --- | --- | --- | | --- | --- | --- |
@@ -769,54 +818,147 @@ Application defaults do not participate in this operation.
| Any other pair of present nodes | Any other pair of present nodes | `compose.conflict` | | Any other pair of present nodes | Any other pair of present nodes | `compose.conflict` |
Every non-mapping overlap is a conflict, even when values are equal, except for Every non-mapping overlap is a conflict, even when values are equal, except for
sequence/sequence contributions at a registered sequence-composition path. Nulls sequence/sequence contributions at a destination with effective identity from an
remain atomic conflicts. Unregistered sequences conflict when repeated and are application registration, a current YAML `key` declaration, or retained lower
never joined by index. Disjoint entries under shared mappings compose, making policy. Nulls remain atomic conflicts.
stable keyed mappings the natural representation for independently contributed Sequences without effective identity conflict when repeated and are never joined
entries. by index. Disjoint entries under shared mappings compose, making stable keyed
mappings the natural representation for independently contributed entries.
`Definition.SequenceCompositions` registers literal mapping paths and one literal `Definition.SequenceIdentities` contains registrations created by
`IdentityField` for sequences that should compose independently authored records. `SequenceIdentityAt` or `SequenceIdentityMatching`. The exact form accepts a
Registration paths follow input-destination key rules: parsed numeric tokens are mapping-only `Path`; the matching form accepts literal-key and any-mapping-key
literal mapping keys, explicit index segments and reserved `configkit` or segments created by `SequenceLiteralKey` and `SequenceAnyMappingKey`. A literal
mapping-valued `extends` segments are invalid, and the root registration is `"*"` key is distinct from the wildcard constructor. Parsed numeric path tokens
harmless and unused. The identity field is one literal string key; neither the remain literal mapping keys; explicit index segments and reserved `configkit` or
field name nor identity strings are normalized. mapping-valued `extends` segments are invalid. The root selector is harmless under
At each registered path, every actual sequence contribution in a strict tier is the mapping-root schema. Selector segments consume one mapping level and match a
validated, including a singleton contribution: each record must be a mapping with complete destination; an any-mapping-key segment matches exactly one key, not an
a present string identity field. Distinct exact identity strings append in source arbitrary prefix or suffix. Each registration requires one or more literal,
and list order; duplicate identities conflict even if the records are otherwise direct field names; compound field sets are normalized as sets, and field names
equal or disjoint. Empty identity strings are valid. Malformed records produce are not otherwise normalized. The same field-set and scalar-equality rules apply
`compose.invalid`; duplicate identities produce `compose.conflict`, with the to YAML `key` declarations.
sequence container and source locations identifying a duplicate without exposing
identity values. Missing paths and non-sequence values do not fail by themselves.
Identical registrations coalesce; conflicting registrations at one path and
invalid registration paths produce `options.invalid`. Unused and root registrations
are allowed and have no effect.
Registration changes strict composition only. Each underlay group and the main At each selected path, records must provide every effective identity field as a
tier has independent duplicate-identity state. Precedence assembly is unchanged: scalar string, number, boolean, or explicit null. Field names form a set, so
a present higher-priority sequence replaces the lower sequence as one value, order and repetition do not affect compound identity. The type-tagged tuple
including at a registered path. Overlays, CLI overrides, defaults, inheritance, preserves category distinctions: strings compare exactly, including empty
and application resolution do not run the registered identity check. The accepted strings; numbers, booleans, and null remain distinct categories. Finite numbers
rationale is recorded in [ADR 0014](adr/0014-sequence-placement-and-composition.md). compare by exact mathematical value, so `1`, `1.0`, and `1e0` are equal, as are
The [maintained prompt catalog](consumers/README.md#identified-prompt-sequences) `0` and `-0`, without floating-point rounding. Distinct tuples append in source
shows item path and glob imports, helper-supplied identity, and main-tier and list order; duplicate tuples conflict even when records are otherwise equal
replacement of a lower-priority sequence. or disjoint. Malformed records produce `compose.invalid`; duplicate identities
produce `compose.conflict`, with paths and source locations identifying a
duplicate without exposing identity values. Missing paths and non-sequence values
do not fail by themselves. Equivalent overlapping registrations coalesce.
Incompatible overlaps and invalid registrations produce `options.invalid` before
source acquisition. Unused and root registrations are allowed and have no effect.
Registrations provide identity for strict composition, sequence operations,
inheritance lookup, and final validation. Dynamic selectors apply at concrete
strict-composition, inheritance, and final-result destinations without requiring
the application to enumerate mapping keys. Each strict tier has independent
duplicate-identity state among its peers; layered precedence and inheritance
then apply the collection operation described below.
Final registration checks run in both ordinary and layered loads after effective
secret lookup and before application semantic validation. They examine actual
final typed sequences, including values introduced by defaults, CLI, or recorded
resolution; they do not require path existence or sequence shape. Diagnostics
use concrete collection and item paths and available source locations, without
exposing identity or credential values. The guarantee ends when loading publishes
successfully and does not cover later caller mutations.
YAML-only identity is enforced during structural composition, precedence, and
identified inheritance; it carries no final-result guarantee after typed
defaults, CLI, or application resolution. Applications that need that guarantee
register the identity constraint.
The [maintained catalog examples](consumers/README.md#identified-prompt-sequences)
show strict peers, precedence operations, inheritance, dynamic destinations, and
the distinction between YAML-only declarations and application registrations.
When an enabled source-version mode is selected, declarations are checked in each When an enabled source-version mode is selected, declarations are checked in each
source's own coordinates before placement and composition. Accepted non-main source's own coordinates before placement and composition. Accepted non-main
headers are consumed before strict composition and cannot supply a registered headers are consumed before strict composition and cannot supply a registered
record identity. Final typed decoding and application validation still run on the record identity. Final typed decoding and application validation still run on the
completed configuration. completed configuration.
Ordinary sources, layered inputs, and selected overlays may declare collection
behavior under `configkit.sequences` at the document root. Each entry has a
required literal `destination` and may specify `key` as one string or a nonempty
list of strings, plus a `mode` of `replace`, `append`, or `upsert`. These
controls are removed before application decoding. A declaration destination is
relative to that source's placement, so a mounted fragment's declaration is
mounted with its values. Declarations from strict peers must agree on identity
fields and on any explicitly supplied mode; omission expresses no mode
preference. They apply even when the declaring source has no value at that
destination: a key validates an existing retained sequence or remains pending
until records arrive, while a mode-only declaration affects later combinations
without recombining earlier tiers. A key supplies the identity used by `append`
and `upsert` and is checked against any application registration.
Strict peers at an identified destination first combine distinct records in
source and list order. Duplicate identities among peers or within one
contribution fail under every mode. After strict composition, precedence and
inheritance use one operation for the assembled incoming collection:
| Mode | Behavior at source-precedence and inheritance boundaries |
| --- | --- |
| `replace` | Replace the complete lower sequence with the assembled higher sequence. |
| `append` | Retain lower records and append higher records; duplicate identities fail. |
| `upsert` | Replace matching records in full at their lower positions, retain unmatched lower records, and append new identities. |
Upsert replaces the complete matched record; it does not recursively merge
record fields. Append and upsert require identity. With no explicit mode in the
current tier, an explicit lower mode persists; if none exists, effective identity
selects upsert and its absence selects replace. Inferred modes do not persist.
Explicit modes persist at their destination until changed by a higher explicit
mode, independently of YAML identity lifetime. A declaration alone does not
clear or recombine a collection.
The presence and value kind at a precedence or inheritance boundary determine
whether a collection changes:
| Higher contribution | `replace` | `append` / `upsert` |
| --- | --- | --- |
| Absent | Leave the lower value unchanged. | Leave the lower value unchanged. |
| Explicit `[]` | Clear the sequence. | Contribute no records. |
| Explicit null or another non-sequence value | Replace the lower value, subject to application schema. | Error; the operation requires a sequence. |
An absent lower value can be initialized by an incoming sequence under any mode.
A present lower null or other non-sequence value is not absence: append and
upsert cannot combine with it. An empty sequence still requires a valid mode and
identity where applicable. Null used as an identity component inside a record is
distinct from null used as the collection value.
Declarations govern ordinary strict source composition, including loads with
imports disabled, and each strict scope in layered loading. Before composing a
higher tier's peers, ConfigKit makes applicable lower identity and explicit mode
available; after peer composition, the tier applies its resolved operation
against the lower collection. Selected overlays use the same collection policy
after the base source graph is composed. During mapping-sibling inheritance, a
parent's YAML identity can follow its collection to the child destination, while
the parent's explicit operation stays at the parent destination. The child
destination selects its operation; absent one, effective identity implies
upsert and its absence implies replace. Whole-sequence replacement discards
lower YAML identity and uses the replacing scope's key declaration, if any;
without one, the collection is unkeyed unless an application registration still
supplies identity. Replacement discards policies owned by removed records, while
an independently declared explicit mode and application registration remain
applicable. Item-local
policies follow their owning records: retaining a record retains its rules, while
whole-record upsert replacement discards the displaced record's rules. Matching
identity or reusing an index does not transfer them. These policies do not change
CLI behavior: CLI sequence assignments replace whole sequences after decoding.
A root `configkit.imports` declaration still requires `Options.Imports`;
`NoSource` has no declaration scope.
An empty mapping contributes no child definitions. Two empty mappings compose, and An empty mapping contributes no child definitions. Two empty mappings compose, and
an empty mapping neither clears nor conflicts with another mapping's children. an empty mapping neither clears nor conflicts with another mapping's children.
Shared mapping containers retain all contributing value locations as their composite Shared mapping containers retain all contributing value locations as their composite
base origin. A conflict reports the first claim as its primary location and every base origin. A conflict reports the first claim in
remaining claim as related locations, in deterministic source and declaration [scoped assembly order](#explicit-imports) as its primary location and every
traversal order. Traversal order never authorizes replacement. Composition clones remaining claim as related locations in the same order. Assembly order never
the resulting tree and its ordered claim records rather than mutating or aliasing an authorizes replacement. Composition clones the resulting tree and its ordered
input document. claim records rather than mutating or aliasing an input document.
## Selected overlays ## Selected overlays
@@ -833,7 +975,9 @@ and typed decoding. Overlay bytes, nodes, and source records count toward the sa
aggregate limits as base inputs. Its canonical path must remain within the configured aggregate limits as base inputs. Its canonical path must remain within the configured
root and must not duplicate a file already present in the base graph. root and must not duplicate a file already present in the base graph.
Overlay is intentionally distinct from strict composition: Collection destinations follow the
[collection operation and presence rules](#strict-document-composition) above.
The following table describes ordinary overlay behavior elsewhere:
| Existing lower node | Incoming overlay node | Result | | Existing lower node | Incoming overlay node | Result |
| --- | --- | --- | | --- | --- | --- |
@@ -842,18 +986,24 @@ Overlay is intentionally distinct from strict composition:
| Mapping | Mapping | Recurse by mapping key | | Mapping | Mapping | Recurse by mapping key |
| Any other pair of present nodes | Any other pair of present nodes | Replace the entire lower node | | Any other pair of present nodes | Any other pair of present nodes | Replace the entire lower node |
Scalars, sequences, and null are atomic. Thus an overlay sequence replaces rather Under these ordinary rules, scalars, sequences, and null are atomic. An overlay
than extends a base sequence, `[]` clears it, null is an explicit replacement, and sequence replaces rather than extends a base sequence, `[]` clears it, null is an
cross-kind replacement is allowed when the final application schema accepts it. explicit replacement, and cross-kind replacement is allowed when the final
application schema accepts it.
Replacing a subtree removes all displaced descendants from effective presence and Replacing a subtree removes all displaced descendants from effective presence and
origin data; history retains the displaced definitions and identifies the parent origin data; history retains the displaced definitions and identifies the parent
replacement. Surviving overlay paths have overlay origins, while recursively shared replacement. Surviving overlay paths have overlay origins, while recursively shared
mapping containers retain their contributing locations. mapping containers retain their contributing locations.
An overlay may not contain `configkit` anywhere, contribute the configured schema An overlay may contain only root `configkit.sequences` declarations; overlay
version, or replace an ancestor of that version with a scalar or null. These rules imports and nested directives are rejected. Sequence declarations are removed
apply even if the supplied version text would be unchanged. Overlays cannot mask a before application decoding. At declared or registered sequence destinations,
base parse, composition, or version failure because those checks run first. `replace`, `append`, and `upsert` use the same identity and policy rules as source
composition. Other overlay sequences remain atomic replacements. An overlay may
not contribute the configured schema version or replace an ancestor of that
version with a scalar or null. These rules apply even if the supplied version
text would be unchanged. Overlays cannot mask a base parse, composition, or
version failure because those checks run first.
## Explicit CLI overrides ## Explicit CLI overrides
@@ -865,10 +1015,16 @@ must never contain the raw argument or configured value. General environment
variable overlays are an explicit non-goal: environment access is reserved for variable overlays are an explicit non-goal: environment access is reserved for
typed secret references. typed secret references.
Overrides apply after the selected overlay and before typed defaults/decoding, so Overrides are prepared after source assembly and inheritance, then assigned after
the input precedence is application defaults < composed base < selected overlay < typed defaults and decoding and before application resolution. The order is:
CLI. Recorded resolution, typed secret lookup, and semantic validation follow the defaults and decoded sources < CLI < application resolution. A CLI mapping
effective override tree. Overrides also work with `NoSource`. recursively overlays the decoded mapping; a CLI sequence replaces the complete
decoded sequence regardless of source collection operations. Inheritance and
typed defaulting do not run again after CLI assignment. Earlier parsing,
composition, inheritance, and decoding failures remain errors even if an override
would replace the failing value. Recorded resolution, typed secret lookup, and
semantic validation use the effective post-override values. Overrides also work
with `NoSource`.
Values are converted directly into ConfigKit's owned tree without YAML parsing or Values are converted directly into ConfigKit's owned tree without YAML parsing or
string coercion. Supported values are `nil`, built-in strings and booleans, all string coercion. Supported values are `nil`, built-in strings and booleans, all
@@ -879,7 +1035,8 @@ false, and zero remain intentional; empty sequences replace whole lists; and map
values recursively overlay existing mappings. Map keys are sorted canonically and values recursively overlay existing mappings. Map keys are sorted canonically and
sequence order is preserved. sequence order is preserved.
All records are detached and validated before any is applied. Root and explicit All records are detached and validated before any is applied. They cannot create
`configkit` or `extends` controls. Root and explicit
sequence-index paths are invalid; parsed numeric tokens remain mapping keys. A path sequence-index paths are invalid; parsed numeric tokens remain mapping keys. A path
may create missing mapping ancestors but cannot traverse an existing scalar, null, may create missing mapping ancestors but cannot traverse an existing scalar, null,
or sequence. Duplicate and ancestor/descendant-overlapping records are rejected. or sequence. Duplicate and ancestor/descendant-overlapping records are rejected.
@@ -896,8 +1053,12 @@ unsupported types, nonfinite floats, and limit excesses fail with safe `override
diagnostics. Diagnostics and provenance identify the safe `Flag` source, never raw diagnostics. Diagnostics and provenance identify the safe `Flag` source, never raw
arguments or values. arguments or values.
Override origins are explicit CLI origins. History retains replaced base/overlay Override origins are explicit CLI origins. Resolution receives CLI values with
definitions and records equal-value replacements as well as structural changes. their explicit-presence and source information. Absence-based defaults preserve
those values, while recorded normalization and derivation may change them. History
retains replaced source definitions and records equal-value replacements as well
as structural changes. Secret lookup and validation use post-resolution values
with their effective source locations.
Caller-owned maps and slices are snapshotted during loading and must not be mutated Caller-owned maps and slices are snapshotted during loading and must not be mutated
concurrently with the call. The executable `ExampleOverride` shows translating a concurrently with the call. The executable `ExampleOverride` shows translating a
parser's explicit-set state into records while preserving an explicitly supplied parser's explicit-set state into records while preserving an explicitly supplied
@@ -905,16 +1066,22 @@ zero.
## Explicit imports ## Explicit imports
Imports are opt-in. With `Options.Imports` enabled, a document may reserve its root Imports are opt-in. With `Options.Imports` enabled, a document may use its root
`configkit` mapping for an `imports` sequence. ConfigKit removes that mapping before `configkit` mapping for `imports` and `sequences` controls. Ordinary `Load` may use
application decoding. Each sequence item is a mapping with exactly one nonempty `sequences` without enabling imports, but an `imports` field still requires
string `path` or `glob`, an optional boolean `optional`, and an optional `Options.Imports`. ConfigKit removes recognized root controls before application
`destination` sequence of strings. Every destination string is one literal decoding. Each imports sequence item is a mapping with exactly one `path`,
mapping key and follows the input-destination validation rules above; quote `dir`, or `glob` selector. `path` and `dir` accept a string or a list of strings;
numeric-looking keys so YAML reads them as strings. Null, scalar destinations, `glob` accepts one path pattern string. The optional `required` boolean controls
and non-string members are `import.invalid`, as are all other unsupported shapes whether an empty selection or a missing named path is an error. Its default is
and fields. A `configkit` key below the root is invalid. With imports disabled, false, so missing imports are absent unless `required: true` is specified. The
any occurrence remains `import.disabled`. removed `optional` field is invalid. `destination` is an optional sequence of
strings. Every destination string is one literal mapping key and follows the
input-destination validation rules above; quote numeric-looking keys so YAML
reads them as strings. Null, scalar destinations, and non-string members are
`import.invalid`, as are all other unsupported shapes and fields. A `configkit`
key below the root is invalid. With imports disabled, root imports and nested
`configkit` keys remain `import.disabled`.
The directive key is `configkit`, not the `_configkit` spelling used in v0.1.0. The directive key is `configkit`, not the `_configkit` spelling used in v0.1.0.
There is no legacy alias: `_configkit` is ordinary application data, subject to There is no legacy alias: `_configkit` is ordinary application data, subject to
@@ -923,38 +1090,107 @@ directive keys in root files and imported fragments, and rename any application
fields previously named `configkit` to avoid the newly reserved key. fields previously named `configkit` to avoid the newly reserved key.
Path declarations use slash-separated relative paths. Absolute paths and network Path declarations use slash-separated relative paths. Absolute paths and network
URLs are invalid. Each path resolves from its declaring file's canonical directory, URLs are invalid. Each path resolves from its declaring file's canonical directory.
and files are traversed depth-first in declaration order: a file contributes its own An empty `path: []` selects nothing; it succeeds even when `required: true` because
body before its imports. Order stabilizes source records and diagnostics but never there are no named targets to be missing. A nonempty required path list verifies
permits replacement. Required missing paths fail; `optional: true` suppresses only every named target, including targets later excluded from selection.
target nonexistence, not invalid paths, containment failures, directories,
permissions, parse errors, or composition errors. Directory declarations search the listed literal directory roots. Missing roots
contribute no candidates; supplied inaccessible or invalid roots fail. `pattern`
accepts one basename pattern or a list, matching filenames only. Omission selects
all filenames; an explicit empty list selects none. `exclude` accepts one basename
pattern or a list and removes matching final candidates. Patterns use Go's
case-sensitive `path.Match` grammar. `recursive` enables descendant traversal;
`depth` is valid only when recursion is effective, with direct files at depth 1 and
depth 0 or omission meaning unlimited traversal subject to library budgets.
`follow` controls discovered symlink entries and `hidden` controls discovered
names beginning with a dot. Both default to true. Explicit roots retain their
normal access behavior.
Glob declarations use the slash-based Go `path.Match` grammar: `*`, `?`, character Glob declarations use the slash-based Go `path.Match` grammar: `*`, `?`, character
classes, and backslash escapes are supported, while recursive `**` syntax is invalid. classes, and backslash escapes are supported, while recursive `**` syntax is
Backslashes escape pattern characters rather than acting as directory separators. invalid. Backslashes escape pattern characters rather than acting as directory
ConfigKit validates the complete pattern before filesystem traversal, walks only separators. ConfigKit validates the complete pattern before filesystem traversal,
directories selected by preceding segments, and reads wildcard directories in walks only directories selected by preceding segments, and reads wildcard
bounded batches. Every examined entry consumes the separate visit budget, including directories in bounded batches. Every examined entry consumes the separate visit
nonmatches. Matches are sorted lexicographically by slash-relative spelling within budget, including nonmatches. Literal components are explicit path components;
each declaration before depth-first source traversal. A required zero-match glob is `follow` and `hidden` filter only wildcard-discovered entries. `exclude` filters
`import.missing`; `optional: true` suppresses only zero matches. Invalid patterns, final candidate basenames.
walk failures, matched directories, invalid documents, escapes, and repeated targets
remain errors. Final candidates are bounded by the remaining source limit before
their contents are accumulated.
Each path or glob declaration may include `placement: mapping` or `Options.ImportDefaults` and `LayeredOptions.ImportDefaults` provide one
application-owned load-wide policy for enabled imports in the main source,
underlays, and transitive imports; they do not apply to selected overlays or
discover sources by themselves. Each applicable field resolves independently:
an explicit declaration value wins, then the application default, then the
library default (`required: false`, `follow: true`, `hidden: true`,
`recursive: false`, and unrestricted filenames). A declaration's `pattern`
replaces rather than extends the default pattern. Nil `Pattern` leaves directory
filenames unrestricted; a non-nil empty slice selects none. Defaults do not make
a field applicable to selectors that do not support it, and load calls snapshot
the supplied pointers and slice before source acquisition.
A source's `configkit.sequences` declarations apply independently of import
enablement; a `configkit.imports` declaration still requires `Options.Imports`.
Sequence identities can be supplied either by these YAML declarations or by
application registrations. The [strict composition contract](#strict-document-composition)
defines their agreement, operation, and lifetime rules.
Acquisition visits declarations in authored order and each declaration's selected
paths in namespace-path lexical order, preserving source-record and version-callback
order. Ordinary mapping contributions in a main tier are assembled dependency-first:
ConfigKit selects the lexically smallest stable source key among the contributions
whose imports are complete. Independent branches may interleave, and the main body
may be assembled after its imports. Sequence-item contributions in that tier are
nodes in the same schedule; each selected item first assembles its own strict
mapping scope, with helpers before the bodies that import them. Those local scopes
do not use application collection policies and are not flattened into the outer
schedule. Item positions and diagnostic paths follow the resulting assembly order.
Each underlay group likewise acquires its inputs before dependency-ready assembly
in one strict scope. Independent groups remain separate precedence tiers; at a
declared or registered destination, the assembled higher sequence uses the
resolved collection operation against the lower sequence. Other sequences retain
whole-value replacement between tiers. Assembly order never permits replacement.
Acquisition order and resolution order serve different purposes: acquisition
assigns source records and callback order, while the shared scheduler determines
composition order among currently dependency-ready contributions. Inheritance
uses the same eligibility rule within each destination-local sibling scope; it
does not reorder the authored output mapping. The private namespace and stable-key
construction is described in the [import implementation overview](internal/imports.md).
An empty selection is allowed by default. `required: true` makes an empty directory
or glob selection fail with `import.missing`; it also makes missing named paths
fail. Requiredness does not suppress invalid paths, containment failures,
directories selected as files, permissions, parse errors, or composition errors.
For inspection, `SourceInfo.ImportSelections` records one outcome for each
completed optional declaration whose entire selected set is empty. The record is
attached to its declaring source, keeps the declaration path local to that source,
identifies the declaration mapping (including its `SourceID`) and selector, and
uses outcome `none`. Records remain in declaration order and are available
through the existing result, load
error, and callback source snapshots. They do not enumerate candidates, explain
why a path was absent, or describe partial omissions. Empty `path: []` succeeds
regardless of requiredness: it produces a `none` record when effectively optional,
and no record when effectively required. Snapshots detach the record paths and
slices.
Each declaration may include `placement: mapping` or
`placement: sequence-item`. Omission means `mapping` for that declaration, `placement: sequence-item`. Omission means `mapping` for that declaration,
including every match of a glob; placement is not inherited. Sequence-item including every match of a glob; placement is not inherited. Sequence-item
placement requires a nonempty effective destination and wraps the imported placement requires a nonempty effective destination and wraps the imported
mapping as one element of a sequence there. It does not split source sequences mapping as one element of a sequence there. It does not split source sequences
or infer identity fields. Repeated sequences still conflict unless their path is or infer identity fields. Repeated sequences combine only when the destination
registered through `Definition.SequenceCompositions`; registration does not has an applicable YAML identity declaration or application registration; source
change placement or precedence behavior. Sequence records retain source placement remains independent of collection identity and operation. Sequence
locations through placement and history; indexes describe their assembled records retain source locations through placement and history; indexes describe
positions while source coordinates continue to identify the declaring files. their assembled positions while source coordinates continue to identify the
Header/version policy and final application validation retain their existing declaring files.
owners and timing; sequence registration is a strict-composition rule only. Header/version policy retains its existing owner and timing. Application
registrations also enforce identity on the final effective typed configuration;
YAML-only declarations are enforced at structural combination and inheritance
boundaries. The [configuration contract](#strict-document-composition) defines
these validation boundaries.
An item import first opens a local mapping scope. Its body and ordinary helper An item import first opens a local mapping scope. Its body and ordinary helper
imports compose within that record. An omitted helper destination or imports compose within that record. An omitted helper destination or
@@ -964,8 +1200,8 @@ the record's `id`. An explicit nested `sequence-item` import creates a sequence
inside the record, and its placement is independent of the enclosing item. inside the record, and its placement is independent of the enclosing item.
At the record root, an item import must name a nonempty nested destination. At the record root, an item import must name a nonempty nested destination.
Physical paths continue to resolve from the file that declares each import. Physical paths continue to resolve from the file that declares each import.
An actual empty mapping contributes an empty item, while an optional unmatched An actual empty mapping contributes an empty item, while an unmatched declaration
path or glob contributes nothing. with default or explicit `required: false` contributes nothing.
For host files, the configuration root defaults to the selected file's canonical For host files, the configuration root defaults to the selected file's canonical
directory. A host file may supply `FilesystemContext.Root` to widen that root but directory. A host file may supply `FilesystemContext.Root` to widen that root but
@@ -984,8 +1220,13 @@ ConfigKit canonicalizes symlinks,
checks containment with relative paths, and opens the canonical root-relative target checks containment with relative paths, and opens the canonical root-relative target
through a library-owned root handle. Contained relative or absolute symlink targets through a library-owned root handle. Contained relative or absolute symlink targets
are accepted; escapes are rejected. Canonical paths define identity, so hard links at are accepted; escapes are rejected. Canonical paths define identity, so hard links at
distinct paths remain distinct sources. Native filesystems use slash-path operations distinct paths remain distinct sources. Native filesystems use slash-path
and their own symlink behavior; imports stay in their selected namespace and root. operations. Where the supplied `fs.FS` implements `fs.ReadLinkFS`, ConfigKit uses
that capability to inspect and resolve discovered links; otherwise it follows
the filesystem's ordinary `fs.FS` behavior. ConfigKit does not require every
native filesystem to expose link inspection, and native filesystem behavior is
not a general sandbox guarantee. Imports stay in their selected namespace and
root.
An active-stack repeat is `import.cycle` and reports the complete cycle's import An active-stack repeat is `import.cycle` and reports the complete cycle's import
sites. A previously visited canonical path is `import.repeated` and reports both sites. A previously visited canonical path is `import.repeated` and reports both

View File

@@ -21,7 +21,7 @@ sanitized shapes using only ConfigKit and the Go standard library:
| Named presets | One-load sibling inheritance, explicit false, nested destination resolution, and source/derivation inspection | Choosing and executing a preset | | Named presets | One-load sibling inheritance, explicit false, nested destination resolution, and source/derivation inspection | Choosing and executing a preset |
| Effective configuration diff | Same-type frozen effective comparisons for two selected files or base-versus-overlay impact | Source/options selection and consumer-owned safe presentation | | Effective configuration diff | Same-type frozen effective comparisons for two selected files or base-versus-overlay impact | Source/options selection and consumer-owned safe presentation |
| Shared source-version checks | One `Definition.Version` policy for accepted declarations across participating source documents, with optional or required declarations and sanitized evidence | Choosing which declarations the application accepts; final assembled schema and domain validation | | Shared source-version checks | One `Definition.Version` policy for accepted declarations across participating source documents, with optional or required declarations and sanitized evidence | Choosing which declarations the application accepts; final assembled schema and domain validation |
| Identified prompt sequences | Explicit item path/glob imports, helper-supplied IDs, reusable single-field registration, and main-tier replacement of a lower-priority collection | Prompt meaning and final domain validation | | Sequence catalogs | Declared collection operations, record inheritance, typed CLI and resolution, inspection, exact and wildcard application registrations | Catalog meaning and final domain validation |
The YAML under [`testdata`](../../examples/consumers/testdata) is loaded by the The YAML under [`testdata`](../../examples/consumers/testdata) is loaded by the
executable examples and ordinary tests. It contains no private application data, executable examples and ordinary tests. It contains no private application data,
@@ -55,6 +55,24 @@ both without redefining their syntax. ConfigKit overlays are configuration overl
they do not select or replace an application's LLM, model, pipeline, report, or they do not select or replace an application's LLM, model, pipeline, report, or
execution profile concepts. execution profile concepts.
## Import defaults and selection inspection
`Options.ImportDefaults` and `LayeredOptions.ImportDefaults` apply one
load-wide policy to enabled main, underlay, and transitive imports. Each field
is resolved independently: a declaration's explicit value wins, then the
application default, then the library default. `Required`, `Follow`, `Hidden`,
and `Recursive` use nil pointers for unspecified values. `Pattern` applies to
directory searches: nil leaves filenames unrestricted, while a non-nil empty
slice selects no files. A declaration's own `pattern` replaces the application
default. The defaults do not enable imports or change selected-overlay behavior.
`Result.Sources`, `LoadError.Sources`, and `Resolution.Source` expose each
declaring source's `ImportSelections`. These records identify the source-local
declaration path, its authored location and selector, and the `none` outcome;
they do not include candidate names or a guessed reason for absence. Returned
records are detached snapshots. The [configuration contract](../config.md#explicit-imports)
owns selector applicability and exact absence semantics.
## Shared source-version checks ## Shared source-version checks
[`LoadVersionedService`](../../examples/consumers/source_version.go) demonstrates [`LoadVersionedService`](../../examples/consumers/source_version.go) demonstrates
@@ -166,21 +184,35 @@ import, source-location, and ownership behavior.
## Identified prompt sequences ## Identified prompt sequences
[`LoadPromptCatalog`](../../examples/consumers/prompt_catalog.go) uses one [`LoadPromptCatalog`](../../examples/consumers/prompt_catalog.go) shows the
definition registration for the literal `prompts` path and `id` field. Its main smaller import-focused adoption path: a registered `prompts.id` identity,
file imports one prompt by path and others by glob, placing each source as an directory and glob selection, a helper import that supplies an item's ID, and
item. A nested ordinary helper import supplies an item's ID. The main tier resource lookup through the effective source context. Its executable examples
strictly appends the distinct records in contribution order; its completed and tests are in
sequence replaces the lower-priority default collection as one value. [`prompt_catalog_test.go`](../../examples/consumers/prompt_catalog_test.go), with
complete [catalog](../../examples/consumers/testdata/prompt-catalog/config.yml)
and [required-file](../../examples/consumers/testdata/prompt-catalog/simple.yml)
fixtures.
The example's `validatePromptCatalog` remains application-owned. It checks that [`LoadSequenceCatalog`](../../examples/consumers/sequence_catalog.go) integrates
prompt names and message lists satisfy this consumer's requirements after an underlay declaration, imported main fragments, identified-record inheritance,
ConfigKit has composed, projected, and decoded the configuration. It does not selected overlay `append`, `upsert`, and `replace`, a typed CLI override, and
add recursive record merging, identity-based overlay, or final uniqueness checks recorded application resolution. Its tests and runnable example demonstrate
after defaults and resolution. See the [strict composition contract](../config.md#strict-document-composition), stable output order, whole-record upsert, provenance/history inspection,
[import syntax](../config.md#explicit-imports), and resource lookup for inherited records, and secret-safe explanation in
[ADR 0014](../adr/0014-sequence-placement-and-composition.md) for the exact [`sequence_catalog_test.go`](../../examples/consumers/sequence_catalog_test.go).
boundaries. [`LoadNamespacedMapCatalog`](../../examples/consumers/sequence_catalog.go) adds
an exact root registration and a wildcard namespace registration, showing that
each concrete sequence has an independent identity scope. The same tests show
that a YAML identity declaration without an application registration does not
create final-result identity enforcement.
The examples keep prompt meaning and domain checks in the consuming application.
The [configuration contract](../config.md#strict-document-composition) owns
declaration, identity, operation, ownership, and final-validation semantics;
[sibling inheritance](../config.md#sibling-inheritance) defines record
relationships, and the [CLI contract](../config.md#explicit-cli-overrides) owns
override ordering.
Known module catalogs should use canonical object entries with a module identifier Known module catalogs should use canonical object entries with a module identifier
and typed options. A small union struct is sufficient when the available option and typed options. A small union struct is sufficient when the available option

View File

@@ -59,6 +59,10 @@ Accepted decisions:
- [Keep the public facade root-owned](adr/0006-public-facade.md) - [Keep the public facade root-owned](adr/0006-public-facade.md)
- [Compare frozen effective configuration snapshots](adr/0010-effective-configuration-diff.md) - [Compare frozen effective configuration snapshots](adr/0010-effective-configuration-diff.md)
- [Place inputs at literal application mapping paths](adr/0011-input-destinations.md) - [Place inputs at literal application mapping paths](adr/0011-input-destinations.md)
- [Share dependency-ready ordering](adr/0015-shared-dependency-ready-ordering.md)
- [Separate import selection controls](adr/0016-independent-import-controls.md)
- [Define identity and merge rules for each collection](adr/0017-collection-sequence-policy.md)
- [Apply CLI overrides after inheritance and decoding](adr/0018-cli-after-structural-resolution.md)
For cross-cutting changes, follow every applicable row. Completed implementation For cross-cutting changes, follow every applicable row. Completed implementation
and remediation plans are preserved in Git history at `cb5b435`; use current and remediation plans are preserved in Git history at `cb5b435`; use current
@@ -131,12 +135,13 @@ Current behavior belongs to the configuration contract's
[input destinations](config.md#input-destinations), [input destinations](config.md#input-destinations),
[imports](config.md#explicit-imports), and [imports](config.md#explicit-imports), and
[shared source-version checks](config.md#shared-source-version-checks) sections. [shared source-version checks](config.md#shared-source-version-checks) sections.
Explicit sequence-item placement and registered strict composition are owned by Explicit sequence-item placement, collection operations, identity registration,
the [strict composition](config.md#strict-document-composition) and and final registered-identity validation are owned by the
[imports](config.md#explicit-imports) sections. The [strict composition](config.md#strict-document-composition) section;
[maintained prompt catalog](consumers/README.md#identified-prompt-sequences) [imports](config.md#explicit-imports) owns acquisition and scheduling. The
shows the current consumer boundary, and [ADR 0014](adr/0014-sequence-placement-and-composition.md) [maintained sequence catalog](consumers/README.md#identified-prompt-sequences)
records the decision. shows the consumer boundary, and [ADR 0017](adr/0017-collection-sequence-policy.md)
records the collection decision.
The completed sequence-composition roadmap, implementation plan, and resolved The completed sequence-composition roadmap, implementation plan, and resolved
remediation record are preserved in Git history at `008d2ba` under remediation record are preserved in Git history at `008d2ba` under
@@ -148,6 +153,35 @@ git show 008d2ba:docs/roadmap/implementation.md
git show 008d2ba:docs/roadmap/remediation.md git show 008d2ba:docs/roadmap/remediation.md
``` ```
The completed import-discovery roadmap, implementation plan, and resolved
remediation record are preserved in Git history at `5747ec4` under
`docs/roadmap/`. For example:
```sh
git show 5747ec4:docs/roadmap/imports.md
git show 5747ec4:docs/roadmap/implementation.md
git show 5747ec4:docs/roadmap/remediation.md
```
The subsequent import-integration audit and completed remediation record are
preserved at `6903c9e:docs/roadmap/remediation.md`; retrieve them with
`git show 6903c9e:docs/roadmap/remediation.md`.
The completed sequence-identity/collection-policy roadmap and implementation
plan are preserved in Git history at `d0a605a`:
```sh
git show d0a605a:docs/roadmap/sequences.md
git show d0a605a:docs/roadmap/implementation.md
```
Current behavior belongs to the configuration contract's
[strict composition](config.md#strict-document-composition),
[sibling inheritance](config.md#sibling-inheritance), and
[CLI overrides](config.md#explicit-cli-overrides) sections.
The completed remediation record, including the algorithm-audit findings, is
preserved at `ea46001:docs/roadmap/remediation.md`.
Potential enhancements are tracked in [future features](roadmap/future.md). Potential enhancements are tracked in [future features](roadmap/future.md).
Install the Go version required by `go.mod` or a compatible newer toolchain. Install the Go version required by `go.mod` or a compatible newer toolchain.
@@ -197,8 +231,10 @@ code changes. Use the component overview to find the corresponding implementatio
| --- | --- | --- | | --- | --- | --- |
| YAML syntax, diagnostics, parser limits | [Adapter tests](../internal/yaml/parse_test.go), [source translation](../internal/engine/parse_test.go) | `go test ./internal/yaml` and `go test ./internal/engine -run '^TestParseSource'` | | YAML syntax, diagnostics, parser limits | [Adapter tests](../internal/yaml/parse_test.go), [source translation](../internal/engine/parse_test.go) | `go test ./internal/yaml` and `go test ./internal/engine -run '^TestParseSource'` |
| File/reader access, imports, globs, containment | [Sources](../internal/engine/source_test.go), [FIFO](../internal/engine/source_fifo_test.go), [imports](../internal/engine/imports_test.go), [relative destinations](../internal/engine/destination_import_test.go), [globs](../internal/engine/glob_test.go) | `go test ./internal/engine -run 'Source\|Import\|Glob\|Filesystem\|Nonregular\|RegularFile'` | | File/reader access, imports, globs, containment | [Sources](../internal/engine/source_test.go), [FIFO](../internal/engine/source_fifo_test.go), [imports](../internal/engine/imports_test.go), [relative destinations](../internal/engine/destination_import_test.go), [globs](../internal/engine/glob_test.go) | `go test ./internal/engine -run 'Source\|Import\|Glob\|Filesystem\|Nonregular\|RegularFile'` |
| Import defaults, empty-selection evidence, and dependency-ready ordering | [Defaults and facade evidence](../facade_import_defaults_test.go), [engine import evidence](../internal/engine/import_evidence_test.go), [scheduler](../internal/engine/dependency_order_test.go), [scheduling integrations](../internal/engine/imports_test.go) | `go test ./internal/engine -run 'Import|ScheduleDependencies|Inheritance'`, `go test . -run 'ImportDefaults|FacadeImport'`, and `go test ./examples/consumers` |
| Native filesystem and mapping sources | [Native acquisition](../internal/engine/native_filesystem_test.go), [filesystem facade](../facade_filesystem_test.go), [mapping facade](../facade_mapping_test.go) | `go test ./internal/engine -run 'NativeFilesystem'` and `go test . -run 'TestFacade'` | | Native filesystem and mapping sources | [Native acquisition](../internal/engine/native_filesystem_test.go), [filesystem facade](../facade_filesystem_test.go), [mapping facade](../facade_mapping_test.go) | `go test ./internal/engine -run 'NativeFilesystem'` and `go test . -run 'TestFacade'` |
| Composition, selected overlays, CLI precedence | [Composition](../internal/engine/compose_test.go), [overlay](../internal/engine/overlay_test.go), [selected overlays](../internal/engine/selected_overlay_test.go), [overrides](../internal/engine/override_test.go) | `go test ./internal/engine -run 'Compose\|Overlay\|Override'` | | Composition, selected overlays, CLI precedence | [Composition](../internal/engine/compose_test.go), [overlay](../internal/engine/overlay_test.go), [selected overlays](../internal/engine/selected_overlay_test.go), [overrides](../internal/engine/override_test.go) | `go test ./internal/engine -run 'Compose\|Overlay\|Override'` |
| Sequence identity, declarations, and operations | [Identity](../internal/engine/sequence_identity_test.go), [registrations](../internal/engine/sequence_registration_test.go), [declarations](../internal/engine/sequence_declaration_test.go), [combination](../internal/engine/sequence_combine_test.go), [final facade validation](../facade_final_sequence_identity_test.go), and [consumer catalog](../examples/consumers/sequence_catalog_test.go) | `go test ./internal/engine -run 'Sequence\|Collection'`, `go test . -run 'SequenceIdentity\|SequenceRegistration'`, and `go test ./examples/consumers` |
| Sibling inheritance, provenance, history, sensitivity | [Expansion](../internal/engine/inheritance_test.go), [destination scopes](../internal/engine/inheritance_walk_test.go), [origins](../internal/engine/inheritance_origin_test.go), [history](../internal/engine/inheritance_history_test.go), [secrets](../internal/engine/inheritance_secret_test.go), [public loading](../facade_inheritance_test.go) | `go test ./internal/engine -run 'Inheritance\|Inherited'` and `go test . -run '^TestFacadeInheritance\|^TestFacadeInheritedSecrets'` | | Sibling inheritance, provenance, history, sensitivity | [Expansion](../internal/engine/inheritance_test.go), [destination scopes](../internal/engine/inheritance_walk_test.go), [origins](../internal/engine/inheritance_origin_test.go), [history](../internal/engine/inheritance_history_test.go), [secrets](../internal/engine/inheritance_secret_test.go), [public loading](../facade_inheritance_test.go) | `go test ./internal/engine -run 'Inheritance\|Inherited'` and `go test . -run '^TestFacadeInheritance\|^TestFacadeInheritedSecrets'` |
| Layered loading and source equivalence | [Grouped lifecycle](../internal/engine/load_request_test.go), [layered facade](../facade_layered_test.go), [equivalence](../facade_equivalence_test.go) | `go test ./internal/engine -run '^TestGrouped'`, `go test . -run '^TestFacadeLayered'`, and `go test . -run '^TestEquivalentSourceForms'` | | Layered loading and source equivalence | [Grouped lifecycle](../internal/engine/load_request_test.go), [layered facade](../facade_layered_test.go), [equivalence](../facade_equivalence_test.go) | `go test ./internal/engine -run '^TestGrouped'`, `go test . -run '^TestFacadeLayered'`, and `go test . -run '^TestEquivalentSourceForms'` |
| Input and relative-import destinations, mounted sources, and source ownership | [Destination engine tests](../internal/engine/destination_test.go), [tier tests](../internal/engine/destination_tiers_test.go), [import traversal and version tests](../internal/engine/destination_import_test.go), [accounting tests](../internal/engine/destination_accounting_test.go), [lifecycle inspection tests](../internal/engine/destination_import_lifecycle_test.go), [public source/error tests](../facade_destination_test.go), [resource contexts](../facade_resource_context_test.go), and [consumer examples](../examples/consumers/destinations_test.go) | `go test ./internal/engine -run 'Destination|Import|Glob|Mounted'`, `go test . -run 'Destination|Import|SourceContext|FacadeMounted'`, and `go test ./examples/consumers` | | Input and relative-import destinations, mounted sources, and source ownership | [Destination engine tests](../internal/engine/destination_test.go), [tier tests](../internal/engine/destination_tiers_test.go), [import traversal and version tests](../internal/engine/destination_import_test.go), [accounting tests](../internal/engine/destination_accounting_test.go), [lifecycle inspection tests](../internal/engine/destination_import_lifecycle_test.go), [public source/error tests](../facade_destination_test.go), [resource contexts](../facade_resource_context_test.go), and [consumer examples](../examples/consumers/destinations_test.go) | `go test ./internal/engine -run 'Destination|Import|Glob|Mounted'`, `go test . -run 'Destination|Import|SourceContext|FacadeMounted'`, and `go test ./examples/consumers` |
@@ -211,6 +247,7 @@ code changes. Use the component overview to find the corresponding implementatio
| Effective configuration diff | [Engine comparison and masking tests](../internal/engine/diff_test.go), [engine operation tests](../internal/engine/diff_operation_test.go), [root facade tests](../facade_diff_test.go), and [consumer examples](../examples/consumers/diff_test.go) | `go test ./internal/engine -run 'Diff|Explain|History|Secret|Independent'`, `go test . -run 'Diff|FacadeInspection|PublicFacadeBoundary|DependencyImportBoundary'`, `go test ./examples/consumers`, and `go test . -run '^ExampleDiff$'` | | Effective configuration diff | [Engine comparison and masking tests](../internal/engine/diff_test.go), [engine operation tests](../internal/engine/diff_operation_test.go), [root facade tests](../facade_diff_test.go), and [consumer examples](../examples/consumers/diff_test.go) | `go test ./internal/engine -run 'Diff|Explain|History|Secret|Independent'`, `go test . -run 'Diff|FacadeInspection|PublicFacadeBoundary|DependencyImportBoundary'`, `go test ./examples/consumers`, and `go test . -run '^ExampleDiff$'` |
| Consumer examples and migration fixtures | [Consumer tests](../examples/consumers/consumers_test.go), [native, layered, and ordinary relative-import examples](../examples/consumers/layered_test.go), [mounted collection examples](../examples/consumers/destinations_test.go), [API examples](../example_test.go) | `go test ./examples/consumers` and `go test . -run '^Example'` | | Consumer examples and migration fixtures | [Consumer tests](../examples/consumers/consumers_test.go), [native, layered, and ordinary relative-import examples](../examples/consumers/layered_test.go), [mounted collection examples](../examples/consumers/destinations_test.go), [API examples](../example_test.go) | `go test ./examples/consumers` and `go test . -run '^Example'` |
| Assembly and layered scaling evidence | [Assembly benchmarks](../internal/engine/assembly_bench_test.go), [layered benchmarks](../internal/engine/underlay_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkUnderlay' -benchmem`; repeat with `^BenchmarkCompose` and `^BenchmarkOverlay` | | Assembly and layered scaling evidence | [Assembly benchmarks](../internal/engine/assembly_bench_test.go), [layered benchmarks](../internal/engine/underlay_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkUnderlay' -benchmem`; repeat with `^BenchmarkCompose` and `^BenchmarkOverlay` |
| Identified-sequence scaling evidence | [Mixed-upsert benchmark](../internal/engine/sequence_combine_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkSequenceMixedUpsert$' -benchmem` |
| Inheritance scaling evidence | [Expansion benchmarks](../internal/engine/inheritance_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkInheritance' -benchmem` | | Inheritance scaling evidence | [Expansion benchmarks](../internal/engine/inheritance_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkInheritance' -benchmem` |
| Documentation links and dependency isolation | [Link checks](../documentation_test.go), [import boundary](../import_boundary_test.go), [public boundary](../public_boundary_test.go) | `go test . -run 'TestLocalDocumentationLinks\|TestDependencyImportBoundary\|TestPublicFacadeBoundary'` | | Documentation links and dependency isolation | [Link checks](../documentation_test.go), [import boundary](../import_boundary_test.go), [public boundary](../public_boundary_test.go) | `go test . -run 'TestLocalDocumentationLinks\|TestDependencyImportBoundary\|TestPublicFacadeBoundary'` |
| Development check behavior | [Development checker test](../development_check_test.go) | `go test . -run '^TestDevelopmentCheckHandlesFilenamesAndStopsOnFailure$'` | | Development check behavior | [Development checker test](../development_check_test.go) | `go test . -run '^TestDevelopmentCheckHandlesFilenamesAndStopsOnFailure$'` |

87
docs/internal/imports.md Normal file
View File

@@ -0,0 +1,87 @@
# Import Discovery and Scheduling Internals
The [configuration contract](../config.md#explicit-imports) owns selector syntax,
defaults, requiredness, placement, inspection evidence, and observable ordering.
This note records implementation flow and private identities for maintainers.
## Discovery and acquisition
Each source's root directive is parsed into presence-aware declarations and
resolved against load-wide defaults. Candidate selection preserves a normalized
namespace path separately from canonical access identity: overlapping roots
coalesce the same candidate spelling in that namespace, while aliases reached
through distinct spellings retain separate inclusion routes for admission.
Directory searches walk each distinct normalized root once, apply all filename
patterns to discovered names, and form a set union of matching candidates across
overlapping roots. Exclusions are applied as set difference after discovery and
candidate normalization. They do not suppress discovery or resource errors.
Directory traversal tracks canonical directories on the active traversal path.
It adds an identity when descending and removes it when the frame is popped, so
only a back-edge to an active directory is a traversal cycle; a later alias is
not mistaken for one. This is separate from import dependency-cycle detection.
For host sources, discovery uses rooted `Lstat` and canonical containment. Native
sources use `fs.ReadLinkFS` inspection when supplied; otherwise traversal follows
the capabilities and behavior of the provided `fs.FS`, without claiming a
general sandbox guarantee.
Acquisition remains authored: declarations are visited in source order and each
selected candidate set in namespace-path lexical order. This preserves source
records, IDs, source-local version checks, and callback ordering. The acquired
inclusion graph is retained separately from composition scopes so cycles that
cross item boundaries remain detectable. Source discovery does not itself choose
sequence assembly order.
## Stable keys and scheduling
The shared scheduler compares immutable tuples component by component using Go's
bytewise string order; if one tuple is a prefix of another, the shorter tuple
sorts first. The import key is
`(namespace-kind, namespace-anchor, resource-kind, resource-path)`:
- Namespace kind is `host` or `native`. Host resource paths are canonical
absolute paths rendered with slash separators; native paths retain their
namespace-relative identity.
- Anchors come from explicit input positions, not discovery order: main is
`(0,0)` and underlay/input positions are `(group+1,input+1)`, each encoded as
fixed-width decimal. Host accesses in one scheduled tier share its host anchor.
A native filesystem namespace uses its earliest explicit selecting position
in that tier. Filesystem tokens distinguish namespace equality, not sort order,
so separately constructed filesystems with the same label remain distinct.
- Resource kind is `file` or `synthetic`. A synthetic root uses its explicit
input anchor and an empty resource path rather than a display label or token
address.
Inheritance keys are
`("definition", scope-pointer, definition-pointer)`, where both pointers use
canonical escaped semantic path spelling. Source IDs are not part of the key.
Both integrations use the same Kahn scheduler: it deduplicates dependency edges,
keeps currently ready nodes in a min-heap, and selects the smallest ready key
after every completion. Its scheduling cost is `O(E + V log V)` time and
`O(V + E)` additional memory, excluding key construction, tuple copying, and
tuple comparisons. On a stalled graph, deterministic DFS over the residual graph
returns a closed cycle rotated to its smallest key; blocked dependents are not
reported as cycle members.
Import composition schedules a main tier or one underlay group as a single
strict-scope graph. Applicable collection declarations are gathered for the
scope before peer composition, including identity inherited from lower tiers.
Identified peer sequences then combine distinct records in assembled source and
list order; after the strict scope completes, layered assembly applies the
resolved collection operation against the lower value, as defined by the
[configuration contract](../config.md#strict-document-composition). Item
contributions own nested local strict-mapping scopes; their helpers complete
before the item is placed in its outer scope. Inheritance uses the same
ready-node rule inside each sibling mapping scope, while preserving the authored
mapping order in its output.
These scopes do not flatten into one global graph, and independent precedence
tiers remain separate.
The main owners are [`import_selection.go`](../../internal/engine/import_selection.go),
[`directory_walk.go`](../../internal/engine/directory_walk.go),
[`import_graph.go`](../../internal/engine/import_graph.go),
[`dependency_order.go`](../../internal/engine/dependency_order.go), and
[`inheritance_schedule.go`](../../internal/engine/inheritance_schedule.go).
Their focused behavior is covered by import-selection, scheduler, and scheduling
integration tests under `internal/engine`.

View File

@@ -9,12 +9,17 @@ Implemented component owners:
- `internal/engine` owns host and native-filesystem source acquisition, - `internal/engine` owns host and native-filesystem source acquisition,
grouped strict composition, recursive precedence assembly, per-input mapping grouped strict composition, recursive precedence assembly, per-input mapping
and sequence-item placement, definition-registered distinct-ID sequence and sequence-item placement, YAML collection declarations and operations for
composition within strict tiers, and immediate-parent-relative import identified sequences, and application identity registrations for strict
placement within that assembly. Higher-priority sequences replace lower composition, inheritance, and final validation. It applies the selected
sequences as whole values. The engine also owns destination-local sibling collection operation at precedence boundaries and owns dependency-ready,
inheritance before typed decoding, plus shared source-version checking with destination-local inheritance before typed decoding. CLI assignment follows
sanitized declaration evidence. inheritance and decoding, before application resolution. The engine also owns
shared source-version checking with sanitized declaration evidence.
Explicit import discovery supports declared paths, directory searches, and
globs; load-wide defaults and source-local empty-selection evidence cross the
root facade without changing acquisition order. Import and inheritance
resolution share dependency-ready scheduling within their composition scopes.
It retains each source's effective placement alongside physical source It retains each source's effective placement alongside physical source
identity and owns schema identity and owns schema
inspection, lifecycle coordination, callback auditing, inspection, lifecycle coordination, callback auditing,
@@ -52,9 +57,10 @@ semantics, including [input destinations](../config.md#input-destinations),
[shared source-version checks](../config.md#shared-source-version-checks), and [shared source-version checks](../config.md#shared-source-version-checks), and
[effective configuration diffs](../config.md#effective-configuration-diffs). [effective configuration diffs](../config.md#effective-configuration-diffs).
The [strict composition contract](../config.md#strict-document-composition) The [strict composition contract](../config.md#strict-document-composition)
owns sequence placement, registration, replacement, and history behavior; the owns sequence declarations, identity, operations, ownership, and final
[consumer guide](../consumers/README.md#identified-prompt-sequences) shows the validation; [CLI overrides](../config.md#explicit-cli-overrides) own override
maintained example. ordering. The [consumer guide](../consumers/README.md#identified-prompt-sequences)
links maintained workflow examples.
The [consumer adoption guide](../consumers/README.md) owns application guidance. The [consumer adoption guide](../consumers/README.md) owns application guidance.
The [architecture policy](../policy/architecture.md) owns contributor The [architecture policy](../policy/architecture.md) owns contributor
boundaries; [ADR 0006](../adr/0006-public-facade.md) records the facade decision boundaries; [ADR 0006](../adr/0006-public-facade.md) records the facade decision
@@ -67,4 +73,10 @@ post-assembly structural operation. [ADR 0012](../adr/0012-relative-import-desti
records the relative-import extension; [ADR 0013](../adr/0013-shared-source-version-checks.md) records the relative-import extension; [ADR 0013](../adr/0013-shared-source-version-checks.md)
records the source-version decision. records the source-version decision.
[ADR 0014](../adr/0014-sequence-placement-and-composition.md) records explicit [ADR 0014](../adr/0014-sequence-placement-and-composition.md) records explicit
sequence placement and composition. sequence placement and composition. [ADR 0015](../adr/0015-shared-dependency-ready-ordering.md)
records the shared dependency-ready ordering decision; [ADR
0016](../adr/0016-independent-import-controls.md) records the independent
selection controls. [ADR 0017](../adr/0017-collection-sequence-policy.md) and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md) record collection
operations and CLI ordering. The [import internals](imports.md) document candidate and
namespace identities, discovery and scheduling, and the bounded algorithms.

View File

@@ -2,7 +2,7 @@
**Base Policy Version:** 1.3 **Base Policy Version:** 1.3
**Last Updated**: 2026-10-16 **Last Updated**: 2026-10-22
**Upstream Source:** [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md) **Upstream Source:** [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
@@ -32,6 +32,12 @@ structure as implemented when its changes land.
Constraints must reflect deliberately selected requirements and decisions. Constraints must reflect deliberately selected requirements and decisions.
Do not promote incidental implementation choices into architectural invariants. Do not promote incidental implementation choices into architectural invariants.
The implemented collection-policy and CLI-ordering constraints below reflect
accepted [ADR 0017](../adr/0017-collection-sequence-policy.md) and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md). The configuration
contract owns their current behavior; these architectural constraints describe
the system boundaries and guarantees that guide changes to that behavior.
## Success Criteria ## Success Criteria
ConfigKit prioritizes a coherent configuration model: consistency and ConfigKit prioritizes a coherent configuration model: consistency and
@@ -91,6 +97,48 @@ boundaries, and keep simple consumers independent of optional composition
features. The [consumer adoption guide](../consumers/README.md) demonstrates the features. The [consumer adoption guide](../consumers/README.md) demonstrates the
implemented surface against representative simple and complex shapes. implemented surface against representative simple and complex shapes.
## Simplicity, Defaults, And Explicit Controls
These principles apply throughout ConfigKit: to configuration syntax, the public
Go API, defaults, and feature design. Evaluate their effects on both configuration
authors and consuming applications.
**Keep straightforward workflows straightforward.** Common configuration
workflows should require little configuration text and minimal API setup.
Support more demanding workflows through explicit, composable options, without
requiring simple consumers to understand or configure those options.
**Prefer simple, permissive defaults with explicit controls.** Choose predictable
defaults that minimize required setup and accommodate valid configurations
without unnecessary restrictions. Provide selectable options for workflows that
need additional capabilities, narrower behavior, or stronger requirements.
Measure simplicity by what consumers must understand and maintain, including
configuration text and application-side plumbing. Concise syntax is useful, but
hidden inference and interacting meanings can make a shorter declaration harder
to understand. Prefer explicitness where it makes behavior more predictable.
A terse configuration format that requires substantial application-side
infrastructure does not satisfy the simplicity principle.
Define what permissiveness means for each behavior. Allowing an optional
contribution to be absent or accepting valid inputs without an unnecessary
filename restriction can reduce required setup. Invalid declarations, ambiguous
composition, and unreadable selected sources must still produce meaningful
errors under the applicable contracts. Permissiveness does not authorize silent
recovery from invalid supplied configuration or weaken configuration integrity.
Options should express coherent choices with understandable purposes and fit
ConfigKit's shared configuration model. Support powerful controls when they
serve concrete needs; these principles do not require a switch for every
implementation behavior or conceivable consumer preference.
Defaults require judgment when simplicity and permissiveness pull in different
directions. Broader discovery can accommodate more layouts while increasing the
implicit search scope. Evaluate such tradeoffs explicitly; these principles do
not mechanically select the broadest behavior for every option. Changes to
existing defaults remain subject to compatibility review, and proposed behavior
remains in the roadmap until implemented.
## Responsibility Boundaries ## Responsibility Boundaries
ConfigKit owns configuration input processing and the coordination of decoding, ConfigKit owns configuration input processing and the coordination of decoding,
@@ -111,6 +159,14 @@ the import extension.
Library code must not depend on consuming applications or initialize their Library code must not depend on consuming applications or initialize their
runtime resources. It must not perform implicit network access, discover runtime resources. It must not perform implicit network access, discover
application-specific paths, or change process-global state during ordinary `Load`. application-specific paths, or change process-global state during ordinary `Load`.
Import discovery is available only through declarations in selected sources:
explicit `path`, `dir`, and `glob` selectors may search their declared scope, but
ConfigKit does not search conventional application paths or discover a main
configuration file implicitly. Imports and sibling inheritance share
dependency-ready scheduling within their existing scopes; acquisition order and
precedence-tier boundaries remain distinct. The [configuration contract](../config.md#explicit-imports)
owns observable behavior; [ADR 0015](../adr/0015-shared-dependency-ready-ordering.md)
and [ADR 0016](../adr/0016-independent-import-controls.md) record the decisions.
The explicit secrets-directory bootstrap is a narrow, opt-in exception: applications The explicit secrets-directory bootstrap is a narrow, opt-in exception: applications
invoke it during startup, and it reports writes, skips and partial failures. It invoke it during startup, and it reports writes, skips and partial failures. It
must never run implicitly as part of configuration loading. must never run implicitly as part of configuration loading.
@@ -133,6 +189,29 @@ The [configuration contract](../config.md#completion-and-resource-lifetime) owns
the precise completion and ownership behavior. The proposed decision record is the precise completion and ownership behavior. The proposed decision record is
[ADR 0008](../adr/0008-complete-on-return-loading.md). [ADR 0008](../adr/0008-complete-on-return-loading.md).
### Accepted Processing Order
Complete source composition and precedence, including selected overlays, before
sibling inheritance. Then apply typed defaults and decode the configuration.
Apply explicit CLI overrides to that configuration and check their types before
running application resolution. Effective secret lookup and final validation
follow resolution, before publication. [ADR 0018](../adr/0018-cli-after-structural-resolution.md)
records this ordering.
CLI overrides are the highest-priority input to application resolution. Supplied
sequences replace whole sequences, and mappings combine recursively. Inheritance
and typed defaulting do not run again after CLI application. CLI paths and values
must not introduce structural directives; earlier source, inheritance, and
decoding failures remain errors even if a CLI assignment would replace the value.
Semantic validation assesses the final configuration after overrides and resolution.
Applications own flag parsing and domain resolution. ConfigKit owns override
application, validation, and source attribution. Resolution receives CLI values
with their explicit-presence and source information. Absence-based defaults
preserve explicit values; recorded normalization and derivation can still change
them. Typed defaults and application resolution retain their own assignment
semantics, independently of collection operations during source assembly.
## Public API And Dependency Direction ## Public API And Dependency Direction
Keep one small, coherent public surface for the supported lifecycle. Simple Keep one small, coherent public surface for the supported lifecycle. Simple
@@ -161,41 +240,44 @@ framework on this library.
## Configuration Integrity ## Configuration Integrity
Within a configuration tier, independent contributions use strict composition: Within a configuration tier, independent contributions use strict composition.
shared mappings combine recursively, while overlapping non-mapping definitions Shared mappings combine recursively. Sequences with an applicable identity
are conflicts, including equal values, except that sequences at explicitly combine distinct records and reject duplicate identities, including equal
registered paths append records with distinct present string identities. Between records. Other overlapping non-mapping values remain conflicts, including equal
precedence tiers, intentional overlay combines mappings recursively, retains values. Conflicting identity or operation declarations among peers also fail.
omitted lower-priority children, Traversal order establishes stable reporting and record order; it does not grant
and replaces other present values according to the existing overlay rules. precedence to peers. [ADR 0001](../adr/0001-separate-composition-and-overlay.md)
Tier precedence is explicit; traversal order within a tier establishes stable records the distinction between strict composition and intentional overrides.
reporting, not permission to override. The distinction and its rationale are
recorded in [ADR 0001](../adr/0001-separate-composition-and-overlay.md).
Sequence-item placement and sequence registration are explicit source and
definition controls; registration does not change precedence replacement. The
[configuration contract](../config.md#strict-document-composition) defines the
behavior, [ADR 0014](../adr/0014-sequence-placement-and-composition.md) records
the decision, and the [consumer guide](../consumers/README.md#identified-prompt-sequences)
shows the implemented subset. This feature does not authorize arbitrary
configurable merge strategies.
The main document and its imports form a strictly composed tier. Explicit The main document and its imports form a strictly composed tier. Explicit
underlay groups compose strictly within each group, then combine recursively underlay groups compose strictly within each group, then combine from lowest to
from lowest to highest beneath the main tier. Source acquisition kind does not highest beneath the main tier. Between tiers, mappings combine recursively and
determine its precedence role. Input-specific constraints, including CLI path retain omitted lower-priority children. Sequence values follow collection policy;
rules and opt-in version ownership, remain part of the configuration contract. Consistent combination other present values retain the established replacement rules. Source acquisition
semantics do not require identical acquisition or lifetime behavior from every kind does not determine precedence. Input-specific constraints, including version
source kind. ownership and path protections, remain part of the configuration contract.
Consistent combination semantics do not require identical acquisition or lifetime
behavior from every source kind.
After source precedence is complete, built-in sibling `extends` relationships Built-in sibling `extends` relationships resolve after source precedence in the
resolve structurally in the same parser-independent representation, using the parser-independent document representation. Preserve existing mapping-sibling
existing recursive overlay rules. This adds neither an acquired source nor a scopes and allow an identified sequence to provide a sibling scope for its
consumer registration mechanism. Enclosing relationships precede nested records. A record may inherit from one parent in that sequence, selected by
destination-local resolution; typed defaults, application resolution, and complete identity. The child supplies and keeps its own identity while inheriting
secret lookup retain their existing lifecycle roles. The [configuration the parent's other settings. Parents remain ordinary records subject to normal
contract](../config.md#sibling-inheritance) owns the exact syntax and presence validation.
rules, and [ADR 0009](../adr/0009-sibling-inheritance.md) records the rationale.
Use the shared dependency ordering and preserve assembled sequence order.
Resolve enclosing relationships before nested destination-local relationships.
Mappings combine recursively; sequences use the inheriting collection's policy.
Whole-record upsert replaces records during combination; explicit inheritance
uses the records that remain. Inheritance remains built-in and creates no
acquired source.
[ADR 0009](../adr/0009-sibling-inheritance.md) records the
original scope and continuing ownership decisions, with sequence combination
and identified-sequence inheritance defined by
[ADR 0017](../adr/0017-collection-sequence-policy.md) and CLI ordering by
[ADR 0018](../adr/0018-cli-after-structural-resolution.md).
ConfigKit must define supported YAML semantics explicitly rather than inherit ConfigKit must define supported YAML semantics explicitly rather than inherit
parser defaults. Processing order and diagnostics must be deterministic for the parser defaults. Processing order and diagnostics must be deterministic for the
@@ -204,13 +286,14 @@ configuration. Do not run application hooks on invalid typed configuration.
User-controlled precedence is application defaults, ordered underlays, composed User-controlled precedence is application defaults, ordered underlays, composed
main input, selected overlay, then explicitly supplied CLI overrides. main input, selected overlay, then explicitly supplied CLI overrides.
Applications own flag parsing; ConfigKit owns overlay semantics and provenance. This priority order is distinct from processing order: typed defaults are applied
Domain resolution follows these at decoding, and application resolution follows CLI input under the lifecycle
inputs and must remain recorded, not become a hidden override channel. rules above.
Preserve node-based composition until the effective input is ready for typed Preserve node-based composition through source assembly and inheritance until
decoding. Preserve presence and source information through processing. Ordinary Go zero typed decoding. Preserve presence and source information through later CLI and
values must not be used to infer whether input was absent. Final configuration, application-resolution changes as well. Ordinary Go zero values must not be used
to infer whether input was absent. Final configuration,
metadata, and effective inspection must agree, including application resolution metadata, and effective inspection must agree, including application resolution
changes; see [ADR 0003](../adr/0003-preserve-presence-and-resolution-history.md). changes; see [ADR 0003](../adr/0003-preserve-presence-and-resolution-history.md).
Safe historical values and an explanation renderer are part of the implemented contract, Safe historical values and an explanation renderer are part of the implemented contract,
@@ -227,6 +310,72 @@ comparison and disclosure behavior, the [component overview](../internal/overvie
owns implementation responsibility, and [ADR 0010](../adr/0010-effective-configuration-diff.md) owns implementation responsibility, and [ADR 0010](../adr/0010-effective-configuration-diff.md)
records the accepted decision. records the accepted decision.
## Collection Identity And Combination
ConfigKit owns a common collection policy for imported records, inline sequences,
selected overlays, and sibling inheritance. Configuration authors may declare
identity and combination rules; applications may register authoritative identity
constraints. Configuration declarations must agree with application constraints.
Keep source placement explicit and separate from policy, retaining the placement
decision in [ADR 0014](../adr/0014-sequence-placement-and-composition.md).
[ADR 0017](../adr/0017-collection-sequence-policy.md) owns the rationale for
collection policy and its relationship to application authority.
Gather declarations for the relevant destination and source or item-local scope
before strict composition. Applicable identity from accumulated lower tiers must
be available when composing higher-tier peers. Higher-tier declarations cannot
repair invalid lower-tier composition. Preserve the distinction between source
acquisition, dependency-ready scheduling within scopes, and precedence order.
Strictly compose each tier's contributions before applying one operation for the
collection against its lower value. Support whole-sequence replacement,
duplicate-rejecting append, and whole-record replacement by identity (`upsert`).
Append and upsert require identity; do not infer it from record fields or position.
Duplicate identities among peers fail under every mode. Source precedence and
inheritance use the same operations and deterministic ordering rules. Whole-record
upsert replaces the matching record's fields together; recursive record merging
and unkeyed sequence concatenation are outside this operation model.
An explicit mode persists at its applicable destination until a higher explicit
mode changes it. With no explicit mode, effective identity selects upsert and
its absence selects whole-sequence replacement. Inferred defaults do not become
persistent instructions. The inheriting destination chooses its operation; copying
a parent collection does not automatically copy its mode to that destination.
Track YAML identity and explicit modes according to their separate lifetimes.
Actual whole-value replacement discards lower YAML identity while preserving
application constraints and any new identity declaration. Item-local policies
belong to their records: retaining a record retains its rules, and discarding it
discards those rules. Policies independently applicable at a destination remain
in effect. Inheritance can copy YAML identity subject to agreement and replacement
rules; application registrations continue to constrain their registered paths.
Numeric positions alone do not establish policy ownership.
Allow declarations without accompanying records. A key declaration validates a
retained sequence immediately or remains pending in its owning scope when no
sequence exists. A mode-only declaration affects later combinations without
recombining earlier tiers. Neither declaration alone clears a collection, and
declaring replace alone does not authorize changing a retained collection's
identity. These rules do not independently impose path-existence or shape
requirements.
Enforce YAML identity during source composition, precedence, and identified
inheritance operations. Application registrations additionally constrain final
effective sequences after defaults, CLI overrides, and application resolution.
The application schema and validation own existence, nullability, and allowed
shape. The final identity guarantee covers actual sequences on successful load;
it does not cover every intermediate callback state or later caller mutations.
YAML-only declarations carry no such final guarantee through defaults, CLI
assignments, or application resolution.
Preserve source attribution, presence, history, sensitivity, and resource context
as records survive, move, or are replaced. Identity diagnostics use paths and
available origin information without exposing identity values. The guarantee
requires no new mutation-history system or exact reconstruction of causes for
records supplied by defaults or callbacks. Exact identity equality, declaration
syntax, presence cases, and operation tables belong in the
[configuration contract](../config.md#strict-document-composition).
## Ownership And Confidentiality ## Ownership And Confidentiality
Returned configuration and metadata must not share mutable state across loads. Returned configuration and metadata must not share mutable state across loads.

111
docs/releases/v0.9.0.md Normal file
View File

@@ -0,0 +1,111 @@
# ConfigKit v0.9.0
This minor release expands import selection, adds author-declared collection
identity and operations, and supports inheritance between identified records.
It also moves CLI assignments after inheritance and typed decoding. These changes
include breaking API and configuration behavior changes from v0.8.0; review the
migration guidance below before upgrading.
## Changes
- Extend imports with named-file and directory lists, directory filename patterns,
exclusions, recursion and depth controls, and explicit hidden-file and symlink
discovery controls. Add `ImportDefaults` to ordinary and layered options, with
authored declarations taking precedence over application defaults. Imports are
optional by default. `SourceInfo.ImportSelections` exposes completed optional
declarations that selected no sources. See the
[import contract](../config.md#explicit-imports) and
[consumer guidance](../consumers/README.md#import-defaults-and-selection-inspection).
- Use shared dependency-ready ordering for import composition and sibling
inheritance. Dependencies complete before dependents, with deterministic
selection among currently eligible contributions. Source acquisition and
callback order remain separate from assembly order. See the
[ordering decision](../adr/0015-shared-dependency-ready-ordering.md).
- Add root `configkit.sequences` declarations for collection identity and
`replace`, `append`, or `upsert` operations, including in selected overlays.
Strict peers combine distinct identities and reject duplicates; precedence
and inheritance apply the destination's collection operation. Upsert replaces
a whole matching record at its retained position. Identity supports compound
field sets and scalar strings, numbers, booleans, and null. See the
[collection contract](../config.md#strict-document-composition).
- Add `Definition.SequenceIdentities`, exact `SequenceIdentityAt` registrations,
and dynamic `SequenceIdentityMatching` selectors. Application registrations
govern structural collection operations and check existing matching final
typed sequences, including values from defaults, CLI assignments, and
application resolution. YAML-only declarations govern structural operations
without adding that final-result check.
- Support record inheritance within an identified sequence using an `extends`
mapping containing the parent's complete identity. The child retains its own
identity, relationships use surviving assembled records, and output sequence
order is preserved. See [sibling inheritance](../config.md#sibling-inheritance)
and the maintained [sequence catalog](../consumers/README.md#identified-prompt-sequences).
- Apply explicit CLI overrides to the decoded configuration before application
resolution. Mappings retain omitted fields and their presence metadata;
sequences replace whole lists independently of collection operations. See the
[CLI contract](../config.md#explicit-cli-overrides).
- Correct native import alias and contained-symlink handling, source-version
path projection, and provenance/history through collection movement,
replacement, inheritance, and CLI assignment. Reduce repeated scans,
accumulated policy copying, and sparse-projection storage during construction.
## Compatibility and upgrade
This pre-v1 minor release intentionally changes the v0.8.0 contracts:
1. **Migrate import requiredness.** The YAML `optional` field is removed and is
rejected. Replace `optional: true` with `required: false`; omission is equivalent
when the application does not default imports to required. Replace
`optional: false` with `required: true`. Imports that previously relied on
implicit requiredness now need `required: true`, or an application
`ImportDefaults.Required` default set to true. Review discovery filters:
`follow` and `hidden` default to true, and directory searches select all
filenames unless a pattern is supplied.
2. **Replace sequence registrations.** `SequenceComposition` and
`Definition.SequenceCompositions` are removed without compatibility aliases.
For a previous registration at `prompts` using `id`, use:
```go
rules.SequenceIdentities = []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
}
```
These registrations now also enforce identity in final typed sequences. Fix
missing or duplicate identities introduced by defaults, CLI assignments, or
application resolution. Registration does not require a collection to exist
or determine its allowed shape; schema and application validation retain
those responsibilities.
3. **Review collection precedence and ordering.** An identified collection now
defaults to upsert across layers and inheritance when no explicit mode
applies, rather than unconditional whole-sequence replacement. Declare
`mode: replace` at the relevant destination where replacement is intended.
With append or upsert, `[]` contributes no records and does not clear the
collection; null is invalid. Use `mode: replace` with `[]` to clear it.
An explicit mode persists at its destination until changed; a parent's mode
does not transfer to a different inheriting destination. Unidentified
sequences retain strict peer conflicts and whole-value precedence
replacement. Dependency-ready assembly can change sequence positions and
diagnostic or history order compared with the former traversal order.
4. **Review CLI recovery and inheritance assumptions.** CLI overrides can no
longer repair values that fail earlier decoding, bypass source or inheritance
errors, or add or cancel inheritance relationships. Supply structural
directives in source configuration. Application resolution still receives
explicit CLI values and may record subsequent normalization or derivation.
5. **Update affected Go literals and diagnostic handling.** Use keyed literals
for expanded option and source-information structs. Collection validation adds
`CodeSequenceInvalid` (`sequence.invalid`) and `CodeSequenceConflict`
(`sequence.conflict`); update diagnostic handling where applicable.
Select this version and rebuild:
```sh
go get gitea.maximumdirect.net/eric/ConfigKit@v0.9.0
```
The consumer minimum remains Go 1.26.0. The YAML dependency remains
`go.yaml.in/yaml/v4 v4.0.0-rc.6`, and the maintainer tool dependency remains
ReleaseKit v0.4.1. Linux amd64 and arm64 remain release-validated targets;
macOS remains best-effort. Publication follows the existing
[release procedure](../release.md).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/docs/oss/attribution.md)

222
docs/roadmap/complexity.md Normal file
View File

@@ -0,0 +1,222 @@
# Internal Complexity Refactoring Roadmap
Status: proposed target state. This roadmap defines a behavior-preserving
refactor; a later implementation plan will define stages and exact private
signatures and file names.
## Purpose
Make ConfigKit's import, inheritance, and projection implementation easier to
understand, review, and change while preserving its public APIs and configuration
behavior. Concentrate on functions that coordinate several responsibilities and
carry substantial shared state.
The target is clearer responsibility and ownership. Adding a few well-named
helpers, private records, or focused files can serve that target even when total
line count increases. Complexity measurements help locate candidates; they do
not establish function-size limits, score thresholds, or release gates.
The [architecture policy](../policy/architecture.md) owns package and ownership
constraints. The [configuration contract](../config.md) and exported Go comments
remain authoritative for consumer behavior.
## Current Baseline
Reviewed baseline: `7a83704f43ef9870db33661231cdff5d6ee502f7`.
The root facade delegates to a cohesive private engine, supported by the ordered
document representation and YAML adapter. Most functions are small. The
complexity audit identified these concentrated maintenance costs:
| Area | Current implementation | Required improvement |
| --- | --- | --- |
| Imports | `imports.go` spans acquisition, item-local composition, and evidence projection. Private loading variants pass long positional argument lists. Source admission and directory selection contain substantial mixed control flow. | Give import entry points explicit, cohesive inputs and separate the responsibilities of acquisition, scope assembly, and evidence projection. |
| Inheritance | `applyInheritanceScopeTemplates` coordinates payload preparation, parent combination, policy ownership, origins, history, and accounting in one executor. | Keep one readable executor with focused operations and explicit state handoffs. |
| Projection | `valueOperation.project` combines type dispatch with struct flattening, map and sequence traversal, scalar conversion, sensitivity, and accounting. | Retain clear dispatch while moving substantial value-kind handling into focused helpers. |
The implemented inventory belongs to the
[component overview](../internal/overview.md). Existing import flow and scheduling
are described in [import internals](../internal/imports.md).
## Target Architecture
The refactor stays within `internal/engine`. The root facade, `internal/document`,
and `internal/yaml` retain their existing responsibilities and dependency
direction. Focused files and ordinary private functions express the new internal
organization. Package extraction is outside this roadmap's selected design.
`executeLoadRequest` remains the visible coordinator of the shared load lifecycle.
Its phase order should remain easy to follow. Helpers expose meaningful work and
results rather than distributing that order across a generic execution framework.
Existing shared mechanisms remain shared: recursive mapping combination,
identified-sequence operations, dependency-ready scheduling, schema descriptors,
and projection/accounting state. New helper boundaries make their ownership and
use clearer without creating parallel implementations.
### Imports: Cohesive Inputs And Scope Ownership
Import loading has a clear private entry surface using named request/context
fields for related inputs. Reuse the existing load-request, acquisition-scope,
and traversal models where they fit. Consolidate forwarding variants whose only
purpose is passing another optional positional argument through the chain.
Retain convenience helpers when they express a useful, distinct operation.
Keep these state lifetimes identifiable:
- Load-wide state: accounting, shared inclusion/repetition information,
namespaces, and source-version processing.
- Acquisition requests: source and filesystem context, placement, declarations,
applicable policy, and the relevant assembly handoff.
- Composition scopes: peer contributions, item-local helpers, record ownership,
and pending declarations.
- Evidence: physical source records, effective destinations, origins, and retained
history projections.
Group acquisition, pending-item assembly, and evidence projection in focused
files. A caller should be able to identify which operation acquires resources,
which composes already acquired content, and which projects retained evidence.
Shared mutable state has a visible owner; a request record need not duplicate
state already held by that owner.
Within dense admission, directory-selection, and declaration-parsing routines,
extract coherent operations where that reduces nested, mixed responsibilities.
Preserve the existing selection algorithms and host/native filesystem behavior.
Resource ownership and closure remain explicit across success and failure paths.
The resulting flow preserves the distinction among acquisition order, strict
scope scheduling, and precedence. Completing a helper scope before placing its
item remains part of the same import flow. Ordinary and layered loading continue
to share the applicable mechanisms.
### Inheritance: One Executor With Explicit Phases
Keep the mapping and identified-sequence adapters and their shared template
executor. The executor coordinates focused operations for:
- Preparing child payload, directives, and destination-local state.
- Preparing the resolved parent payload and applicable inherited policy.
- Combining values through the existing mapping and sequence mechanisms.
- Producing policy, origin, replacement, and replay evidence from that combination.
- Accounting for expansion and retained history under the existing budgets.
These are responsibility boundaries, not a required helper count or a new
scheduler. Small private result records are appropriate when related outputs
otherwise travel through long argument lists or implicit mutations.
Authored templates, resolved values, scoped policy, and inspection evidence remain
distinguishable. Immutable borrowed inputs and owned mutable results have clear
lifetimes. Nested instructions remain available for destination-local evaluation.
For identified-record inheritance, the child's own identity fields and origins
remain intact; the parent's identity fields are excluded from inherited payload
and its history, as the current contract requires.
One sequence-operation result continues to drive value construction, record
ownership, origins, and historical position projection. Helpers consume that
evidence directly. Scheduling remains dependency-ready, enclosing relationships
resolve before nested ones, and assembled sequence order remains unchanged.
### Projection: Explicit Value-Kind Handlers
Keep `projectConfiguration` as the schema-checked entry and `valueOperation` as
the owner of per-operation traversal and accounting state. The central dispatch
clearly selects the appropriate handling for pointers, dynamic values, structs,
maps, sequences, and scalars.
Substantial struct, map, and sequence work lives in focused handlers. Struct
projection retains declaration order, inline flattening, collision diagnostics,
and sensitivity propagation. Map projection retains deterministic key ordering.
Scalar conversion retains the existing number, duration, text-codec, and secret
reference representations.
Depth and node accounting retain their existing meaning, including flattened
containers. Cycle tracking stays active through recursive child traversal and
is balanced on error paths as well as success. Passing state through a helper
does not introduce another full-tree projection or an unnecessary ownership copy.
Cloning and source-value construction remain distinct operations with their
existing semantics. Reuse small helpers when the semantics coincide; keep the
projection refactor centered on making the existing traversal readable.
## Behavior And Scope Preservation
The implementation preserves:
- Root-owned public types, signatures, supported import paths, and facade boundaries.
- Supported source forms, opt-ins, import controls, placement, acquisition and
composition ordering, and source/version evidence.
- Strict peer composition, precedence operations, identity agreement and lifetime,
record ownership, and destination-local inheritance.
- The load lifecycle, including typed defaults, post-decode CLI assignment,
recorded application resolution, and final validation.
- Explicit presence, provenance, history, sensitivity, resource contexts, and
detached publication/query results.
- Established diagnostic codes, semantic paths, source coordinates, ordering,
resource lifetime, and existing limit behavior.
The owning contracts define the extent of these obligations. Incidental private
call sequences, exact helper names, and allocation counts do not acquire new
guarantees through this roadmap. The work adds no consumer options, configuration
syntax, dependencies, or complexity-based gates.
## Relationship To Remediation And Other Work
The completed remediation record at `ea46001:docs/roadmap/remediation.md`
documents the identified correctness and algorithmic efficiency findings.
Several overlap these files, particularly item-scope application matching and
inheritance policy localization.
Use the documented contract and completed remediation as the behavioral baseline.
Keep corrective changes reviewable under their findings, and retain their
regressions through this refactor. The later implementation plan should coordinate
overlapping work and select its starting revision rather than duplicating the
remediation plan here. These structural changes do not close a finding merely
because its function moved or became smaller.
The pending [discovery roadmap](discovery.md) remains separate feature work.
Import cleanup should leave clear private boundaries for future use without
implementing discovery behavior as part of this refactor.
## Work Needed And Validation
Implementation needs to consolidate import inputs and adapters, separate import
responsibilities, extract inheritance operations with explicit outputs, and
decompose projection handling. Remove superseded private plumbing as its callers
migrate. Review the resulting call flow for repeated traversals, policy scans,
copies, or hidden state transitions introduced by extraction.
Reuse the existing behavioral suites under the
[development guide](../development.md) and [testing policy](../policy/testing.md):
- Imports: selection, scheduling, source/version evidence, placement, helper scopes,
ordinary/layered equivalence, filesystem failures, and resource lifetime.
- Inheritance: mapping and record relationships, nested destination resolution,
cancellation and replacement, policy ownership, history, sensitivity, and
independent loads.
- Projection: schema/decoding agreement, inline fields, text codecs, nil/empty
values, cycles and budgets, secrets, and detached results.
Add focused regressions only where a consequential handoff risk lacks coverage.
Tests should continue to describe behavior as private functions and files move.
Use the existing everyday check and applicable ownership race checks during
implementation. Complexity measurements can inform review; acceptance does not
depend on numerical scores or a new benchmark campaign.
Update the component overview and import internals to describe the implemented
ownership and file responsibilities when the refactor lands. Update contributor
navigation and test-owner references where moves affect them. Public behavior
continues to be defined by its current canonical owners.
## Completion Criteria
- Import callers pass cohesive named inputs, with clearly owned load-wide and
scope-local state and fewer redundant forwarding variants.
- Import acquisition, item assembly, and evidence projection are easy to locate
and follow across focused private files.
- Inheritance orchestration exposes value, policy, evidence, and accounting work
without requiring readers to reconstruct their interaction from one large body.
- Projection dispatch and substantial value-kind handlers have clear boundaries
and share the original operation state correctly.
- Existing APIs and contracted behavior, including completed remediation, remain
covered by the relevant behavioral and ownership checks.
- Internal documentation matches the resulting implementation.

381
docs/roadmap/discovery.md Normal file
View File

@@ -0,0 +1,381 @@
# Composable Configuration Discovery Roadmap
Status: proposal with OD-1 through OD-5 and the final review clarifications
settled and integrated below.
This document describes future work, not an implemented contract. A staged
implementation plan will follow review of the complete roadmap.
## Purpose And Ownership
Make ordinary file-based loading concise for applications that want conventional
configuration locations. ConfigKit supplies useful built-in candidate lists and
an ordered selection operation. Consumers continue to own main-file search
conventions: they choose which lists apply, their order, custom candidates, and
whether a configuration file is required.
A consumer can select one preset for a simple application or compose several
presets and custom path lists. The application supplies an explicit slug where
needed to construct conventional paths. ConfigKit does not infer application
identity from the executable name or enable discovery for existing explicit
sources.
Discovery selects one source for the existing loading lifecycle. It does not
create additional configuration tiers or merge every matching file. Existing
layered loading remains available when consumers intentionally combine sources.
## Current Baseline
The [source contract](../config.md#sources) provides explicit host files, native
filesystem files, bytes, readers, mappings, and `NoSource()`. A missing explicit
file fails. The zero source is invalid. There is no conventional main-file search.
[Ordinary loading](../config.md#single-source-loading) infers a selected overlay's
filesystem context from the main source. `NoSource()` currently rejects enabled
imports, active overlays, and filesystem context. In
[layered loading](../config.md#layered-loading), an absent main input can coexist
with underlays; a selected overlay requires an explicit `OverlayFilesystem`.
These distinctions matter when an optional search finds nothing.
The [architecture policy](../policy/architecture.md#responsibility-boundaries)
assigns search conventions to consumers and currently excludes implicit main-file
discovery. The addition should clarify that consumers may explicitly select
library-provided search conventions. Consumer ownership remains unchanged.
## Agreed Direction
### Filename Groups, Directory Searches, And Search Presets
Organize discovery around three composable descriptions. These are ordered lists:
their order determines precedence.
**Filename groups** provide reusable, ordered filename templates. The built-in
groups are:
| Group | Filenames, in order |
| --- | --- |
| Named | `<appname>.yml`, `<appname>.yaml` |
| Generic | `config.yml`, `config.yaml`, `main.yml`, `main.yaml` |
`<appname>` is the application slug supplied by the consumer.
**Directory searches** pair a directory with an ordered list of filename groups.
The default selection is `[Named, Generic]`, producing the six filenames above
in that order. A directory search may select different groups or reverse their
precedence, such as `[Generic, Named]`, while retaining each group's internal
order. Consumers can also supply custom filenames, subject to the literal
single-component rules under OD-5 below.
**Search presets** bundle ordered directory searches. Consumers can select a
complete preset for the simple case or compose presets, custom directory searches,
and explicit file candidates in their desired order. Explicit file candidates
are literal paths and receive no filename expansion.
### Ordered Expansion And Concatenation
The selection model is **ordered expansion and concatenation**, followed by
first-match selection:
1. Expand the selected presets into directory searches in their supplied order.
2. For each directory search, concatenate its selected filename groups in order,
substitute the application slug where needed, and join each filename to that
directory.
3. Concatenate the resulting candidate lists, preserving the positions of custom
directory searches and explicit file candidates.
4. Try candidates in order, stopping at the first selected file or an error as
described below.
Directory priority comes before filename priority: exhaust one directory's
filenames before trying the next directory. When several directories share a
filename list, their expansion is a Cartesian product traversed directory first.
Different directories may select different lists, so ordered expansion and
concatenation describes the complete model. Set union does not express its
precedence semantics.
The eventual documentation should explain this model using the applicable
mathematical nomenclature: ordered lists, ordered concatenation, and Cartesian
product where a shared filename list applies. Retain these names in relevant
implementation comments where they help readers understand the operations. The
model describes ordering without requiring eager construction of every candidate.
### Unix System Preset
The agreed Unix system preset is:
| Directory, in search order | Filename groups, in order |
| --- | --- |
| `/usr/local/etc/<appname>` | `[Named, Generic]` |
| `/etc/<appname>` | `[Named, Generic]` |
| `/etc` | `[Named]` |
This produces 14 candidates: six in each application-specific directory, then
`/etc/<appname>.yml` and `/etc/<appname>.yaml`. For example,
`/usr/local/etc/<appname>/main.yaml` precedes
`/etc/<appname>/<appname>.yml`. Selecting only `Named` for `/etc` is an ordinary
directory-search description; the selection algorithm needs no special exception.
### Additional Presets (OD-1 — Settled)
Provide the following independently selectable presets alongside Unix system.
Within each preset, search directories in the order shown:
| Preset | Directory | Filename groups, in order |
| --- | --- | --- |
| User configuration | `<user-config-dir>/<appname>` | `[Named, Generic]` |
| User configuration | `<user-config-dir>` | `[Named]` |
| Home directory | `<home>/.<appname>` | `[Named, Generic]` |
| Home directory | `<home>` | `[Named]` |
| Working directory | Current working directory | `[Named]` |
Resolve the user-configuration base with
[`os.UserConfigDir`](https://pkg.go.dev/os#UserConfigDir), the home base with
[`os.UserHomeDir`](https://pkg.go.dev/os#UserHomeDir), and the working-directory
base with [`os.Getwd`](https://pkg.go.dev/os#Getwd). Adopt the Go helpers'
platform-specific directory and environment behavior, including their errors;
the settled OD-2 rules below govern how discovery handles resolution errors.
The goal is to support familiar conventions through idiomatic Go. This feature
does not add a separate XDG compatibility layer or XDG system preset. A selected
user-configuration base has no additional hard-coded `<home>/.config` fallback.
The Home directory preset is a separate, explicit choice.
The working-directory preset searches only that directory and uses application-named
files; it does not walk ancestor directories. The consumer chooses precedence
between all presets. Selecting one does not automatically select any others.
### Illustrative Consumer API
Names below are illustrative and are not implemented Go API:
```go
// A simple system-wide application.
source := configkit.SearchConfig("myapp", configkit.UnixSystemDirectories())
```
```go
// A consumer-selected working-directory, user, then system search.
source := configkit.SearchConfig(
"myapp",
configkit.WorkingDirectory(),
configkit.UserConfigDirectory(),
configkit.UnixSystemDirectories(),
)
```
```go
// Explicit custom candidates followed by a selected conventional fallback.
source := configkit.SearchConfig(
"myapp",
configkit.Paths("/run/myapp/config.yml", "/opt/myapp/config.yaml"),
configkit.UnixSystemDirectories(),
)
```
Use the resulting source with ordinary loading or as a layered main input. Custom
searches and file lists participate at their specified position; they do not
silently append unselected presets. Reordering presets or their directory
searches changes search priority.
### Selection And Failure Behavior
| Situation | Selected behavior |
| --- | --- |
| Candidate is absent | Continue to the next candidate. |
| A candidate file is selected | Stop searching and load that file. |
| Candidate is inaccessible, unreadable, or not an accepted file type | Fail using normal source diagnostics. |
| Selected file fails parsing, version checks, composition, decoding, or validation | Fail without trying another candidate. |
| Entire search has no match | Absence is permitted by default; follow the shared absent-main lifecycle settled under OD-3 below. |
| Entire search has no match and is required | Fail with a diagnostic explaining the unsuccessful search. |
| Caller explicitly uses `File(path)` | Retain existing exclusive-file behavior, including errors for a missing file. |
Requiredness applies to the complete search, not to each preset or directory.
Permitting an absent file does not waive application version, schema, or domain
requirements; the remaining lifecycle may still reject an incomplete configuration.
Perform discovery before parsing the chosen main file. Do not implement fallback
by repeatedly running complete loads. Defaults, application resolution, secret
lookup, and final validation retain their existing lifecycle execution rules.
Once a file has been selected, later failure does not restart discovery.
### Construction And Filesystem Behavior
Constructors describe the search without filesystem access. Resolve dynamic base
directories and access candidate files during loading, so reusable descriptions
can observe changes on subsequent loads. Do not change the process working
directory or environment.
Reuse the host-file acquisition rules for selected files, including regular-file
checks, symlink handling, resource ownership, and source limits. The feature does
not require a new filesystem traversal engine, recursive directory search, or
new file-size limits. Candidate filenames and groups are bounded by their
explicit descriptions, rather than by scanning unrelated directories.
The exact Go names, constructors, and representation of search-wide options
belong in the implementation plan. Keep the public surface root-owned and keep
the single-preset case concise.
### Unavailable Bases And Platform Applicability (OD-2 — Settled)
Resolve directory bases only as the ordered search reaches them. Once an earlier
candidate is selected, later searches do not resolve their bases. Static invalid
search arguments still fail ordinary option validation before source acquisition.
An explicitly empty filename selection skips directory resolution as specified
under OD-4; it has no candidates to locate.
If a selected directory's base cannot be resolved, stop and report the error.
A missing directory or candidate file beneath a resolved base allows the search
to continue; a directory-resolution error does not. This distinction also applies
to optional searches: permitting no match does not suppress errors encountered
while searching. Preserve the agreed errors for inaccessible or unreadable
candidates.
Treat using the Unix system preset on an unsupported platform as an explicit
configuration error when that preset is reached. Specify its supported platforms
in the implementation plan. Do not translate the Unix preset into a different
platform's convention. User, home, and working-directory presets use their native
Go directory helpers.
This keeps search priority predictable when directory resolution fails. Consumers
select the presets appropriate to their application and environment; this proposal
does not add an option to skip unavailable bases or unsupported presets.
### No Match, Imports, And Selected Overlays (OD-3 — Settled)
Use a shared absent-main lifecycle for optional no-match and explicit `NoSource()`.
Allow imports to be enabled when the main file is absent: there are no main-file
import declarations to process, and enablement does not introduce automatic
directory scanning. Other actual sources retain their normal import behavior.
In layered loads, process underlays normally and retain the existing explicit
`OverlayFilesystem` requirement for an active overlay.
For ordinary loads with an active overlay and no main file, require an explicit
filesystem context to locate the overlay. Do not infer an overlay root from the
first or last searched directory. A missing selected overlay remains an error;
absence of a main file does not silently disable a requested overlay.
For an absent main input, a host filesystem context specifying only `Root` is
sufficient to locate the overlay; no main-file `BaseDir` is needed. If discovery
selects a host file, retain the normal file-context rules: the selected file
determines its base directory, an explicit `BaseDir` is rejected, and the file
must be contained within any supplied root. This allows one root-only context
to work with both selected and absent main inputs, subject to those existing
root and containment checks. An invalid context is an error, not a reason to
resume discovery.
Defaults, CLI overrides, application resolution, and validation continue through
the normal lifecycle. Main-version checking receives an absent main input and
may reject it under the application's existing policy. Create no synthetic empty
document or source record for the absent main file.
This deliberately relaxes the current `NoSource()` restrictions on imports and
on overlays with explicit context. Both ways of obtaining an absent main input
have the same behavior. Implementation must cover this lifecycle change as well
as candidate selection.
### Empty Composition And Implicit Preset Selection (OD-4 — Settled)
Require explicit preset or search selection to activate a convention. Supplying
no presets or custom searches produces no candidates and activates no implicit
fallback. Empty custom path lists and explicitly empty filename-group selections
also contribute no candidates. An empty component does not prevent later
components from contributing candidates.
A directory search with an explicitly empty filename selection does not resolve
its directory base. Static argument validation still applies. This gives empty
selections their no-candidate meaning without introducing directory-resolution
errors for a search that cannot select a file.
If the complete search is empty, an optional search follows the absent-main
lifecycle settled under OD-3; a required search fails as exhausted. Empty lists
are valid, supporting consumers that construct their searches programmatically.
Distinguish omission from an explicitly empty filename selection for a directory
search. Omitting the selection uses the agreed `[Named, Generic]` default;
explicitly selecting an empty list means no filenames. The eventual API must
represent both intentions clearly. Presets retain their specified filename groups.
Selecting a single preset keeps the simple case concise. Search-wide requiredness
provides the check for consumers that must find a configuration file.
### Slugs And Custom Path Interpretation (OD-5 — Settled)
Require a nonempty single-component application slug whenever a selected search
uses it to construct a directory or filename. Reject path separators, `.` and
`..`, and forms unusable as that component on the target platform. Preserve
spelling and case; do not trim, lowercase, or impose a kebab-case naming scheme.
Validation keeps the slug from changing the intended directory structure without
imposing a separate naming style.
Searches that do not use the slug do not require one. Examples include explicit
file candidates and a custom directory searched with only Generic filenames.
Treat custom paths as literal host paths. Resolve relative paths against the
working directory at load time. Do not expand shell variables, `~`, or wildcard
syntax. Consumers can construct expanded paths themselves; User and Home presets
provide the common directory-resolution conveniences. These rules keep custom
paths consistent with explicit file loading.
Custom filenames within directory searches are literal single filename
components. They cannot be absolute paths or contain subdirectory components;
use custom directory searches or explicit file candidates for those layouts.
They receive no application-name placeholder expansion. The Named group supplies
application-name expansion, while consumers may construct custom filenames
themselves before supplying them.
### Placement, Provenance, And Existing Features
The selected file participates as the actual file source. Preserve its physical
identity, effective placement, source origin, and passive resource context.
Relative imports and relative application resources use the selected file's
normal context. Discovery does not relocate overlay or CLI coordinates.
Missing candidates are not acquired sources and do not create source IDs,
configuration history, or explicit input presence. An entirely absent search
contributes no synthetic empty configuration file. Diagnostics should distinguish
an exhausted required search from a failed candidate. Existing selected-source
inspection remains the foundation; a new public per-attempt trace is not needed
for this proposal.
Preserve collection policies, inheritance, source-version rules, CLI ordering,
final validation, and frozen inspection after selection. Search-list order does
not change precedence among underlays, the main input, an overlay, and CLI values.
An explicitly selected overlay is not another main-file fallback candidate.
## Work Needed From The Baseline
The subsequent implementation plan should cover:
- Define filename-group, directory-search, preset, and search-source representations,
slug validation, and a search-wide requiredness option. Keep construction
separate from acquisition and the simple preset selection concise. Distinguish
omitted and explicitly empty directory-search filename selections.
- Implement ordered expansion and concatenation for presets, filename groups,
custom searches, and explicit file candidates, followed by first-match selection
through existing file access. Resolve each source once per consumption before
entering its parse and composition lifecycle.
- Integrate selected and absent outcomes with ordinary and layered main inputs,
placement, imports, application version checking, and overlay contexts according
to OD-3, including root-only host contexts for absent main inputs and normal
file-context checks when a file is selected. Preserve ownership and close
acquired handles on success and failure.
- Preserve real selected-source metadata and safe diagnostics without treating
absent candidates as configuration contributions.
- Exercise preset, directory, filename-group, and filename ordering, reversed
group precedence, the Unix system preset's 14 candidates and named-only `/etc`
search, custom-list composition, both filename extensions, no-match behavior,
empty components and searches, omitted versus empty filename selections,
skipped directory resolution for empty filename selections, slug-dependent and
slug-independent searches, literal custom paths and filename components,
required searches, unavailable preset bases, errors that must stop fallback,
repeated loads, and untouched explicit `File`
behavior. Cover imports, overlays, underlays, defaults, CLI, source versions,
source contexts, and source-list/inspection evidence at their boundaries.
- Clarify the architecture policy's consumer-selected discovery boundary. Record
any warranted durable ownership decision in an ADR; keep preset filenames and
API mechanics in their canonical contract when implemented. Update consumer
examples and the configuration contract when the feature lands. Explain the
three-part discovery model and its mathematical terminology in the eventual
documentation, with simple preset selection and custom composition examples.
This roadmap does not change implementation, policy, or ADRs by itself. Coordinate
implementation with the ongoing sequence integration audit rather than assuming
its in-progress changes are part of a fixed implementation baseline.

View File

@@ -6,105 +6,19 @@ commitments. Adopting a candidate requires an explicit design decision and a
separate plan. Significant architectural exclusions and their rationale belong separate plan. Significant architectural exclusions and their rationale belong
in ADRs; this document does not maintain a list of rejected proposals. in ADRs; this document does not maintain a list of rejected proposals.
## 1. Identified record sequences ## 1. Recursive body merging for identified records
Status: deferred. PromptKit proposed this capability from its retired proposal 2, Status: deferred. The implemented
**Namespaced Collections As Identified Records**. The original proposal is in the [collection policy](../config.md#strict-document-composition) uses `upsert` to
PromptKit repository at replace a complete matched record. Recursive merging of fields inside matching
`b5866dd:docs/roadmap/configuration-collections-identified-records.md`. records remains a separate candidate and would revisit the whole-record update
PromptKit selected mounted mappings instead; this feature is not a dependency boundary selected by [ADR 0017](../adr/0017-collection-sequence-policy.md).
of that design or a request to restore multiple installed prompt versions.
A narrower [sequence placement and composition feature](../config.md#strict-document-composition) Consider this only in response to a concrete consumer need. A proposal would need
is implemented: consumers can place imported mappings as sequence items and to define conflicts for overlapping body fields, removal of omitted fields,
register a single literal string identity field for duplicate-rejecting strict presence and ordering, and how provenance, history, sensitivity, and resource
composition. Precedence still replaces sequences as whole values. This entry context follow partially retained records. Any change requires a new decision and
tracks broader identified-record behavior that remains deferred and is not part implementation plan; this entry does not alter current upsert behavior.
of that implemented contract.
### Motivation and proposed behavior
Allow applications to declare that selected sequences contain records identified
by explicit fields, while retaining sequence-shaped output for slice decoding
and effective YAML. This could make composite identities and independently
authored catalogs more convenient than encoding identity in nested mapping keys.
Current [composition](../config.md#strict-document-composition) appends distinct
records only at explicitly registered paths; other sequences remain atomic.
Selected [overlays](../config.md#selected-overlays) always treat sequences
atomically.
The proposed broader opt-in registration would extend the current reusable
application definition from one identity field to a composite identity made of
an ordered, nonempty set of distinct direct string-field names. For example,
records at `profiles` could be identified by `(namespace, id)`. Records
`(provider, brief)`
and `(application, brief)` would coexist, and a later layer could update the
provider record's temperature while retaining its backend and model.
The requested semantics, all still subject to design review, are:
- Match exact string tuples, preserving component boundaries and spelling,
including empty strings. Every contribution supplies its complete identity;
defaults, inheritance, filenames, and normalization do not supply it.
- Retain distinct identities. Matching records from strict peers recursively
compose disjoint body fields and conflict on overlapping non-mapping body
values, even equal values. Equal identity fields act as structural keys rather
than conflicts and retain their contributing source evidence.
- Higher layers recursively overlay matching bodies. A changed identity denotes
a different record, not a rename. Proposed ordering is first contribution in
precedence-assembly order; matching records keep position and new ones append.
- Reject malformed identities, non-mapping records, non-null non-sequence values
at registered paths, and duplicate tuples within one source sequence. Later
layers cannot repair malformed structural identity in an acquired contribution.
- Treat an empty registered sequence as contributing no records; whole-path null
retains ordinary conflict/replacement semantics. Unregistered sequences,
including nested message lists, retain existing atomic behavior.
- Support record inheritance within the same registered sequence using an
`extends` mapping containing the complete parent tuple. Resolve only final
active relationships after precedence, preserve the child's identity, overlay
its body on the parent's body, and consume the directive before decoding.
Preserve cancellation, missing-parent/cycle errors, and declaring resource
contexts. Existing string-based mapping-sibling inheritance stays unchanged.
Integrate ordinary and layered loads, imports, mounted paths, selected overlays,
and whole-sequence CLI overrides. Keep origins, presence, history, sensitivity,
and resource contexts attached to the correct record despite different source
and final indexes. Diagnostics must retain source locations without exposing
sensitive identity values. Final inspection may use snapshot-specific sequence
indexes; existing atomic sequence diffs need not become identity-aware.
Successful results would have valid, unique identities, including after defaults
and application resolution, without a second merge or inheritance pass. Existing
source-version checks remain independent; consumed headers cannot supply missing
record identities. No identity-selector path syntax, arbitrary merge callbacks,
wildcard registration, per-record deletion/rename operator, multiple inheritance,
or cross-path parent references are requested.
### Questions to resolve before implementation
- **Need and delivery scope:** does an active consumer benefit enough to justify
the added semantics over keyed mappings? Are composite identities required?
Staging composition before inheritance is possible but would leave part of
the request incomplete.
- **Defaults and callbacks:** do default records merge by identity or retain
existing sequence-replacement behavior? How are callback-created duplicates,
identity edits, and reordering validated and reflected in provenance?
- **Null, empty, and order:** what happens when null clears a collection and a
later layer reintroduces records? How do absent and explicitly empty sequences
interact with defaults and first-contribution order?
- **Inheritance references:** how do peers compose relationship declarations,
and how are complete references enforced without assembling partial tuples
accidentally? What happens if identity fields overlap structural controls?
- **Registration paths:** how do overlapping registrations and paths beneath
sequence indexes behave when assembly changes those indexes?
- **Evidence and lifecycle:** how are record identities mapped between source
trees, history, inheritance, callback snapshots, and final indexes without
mixing records' provenance or sensitivity? Where do identity checks run relative
to header consumption and other structural processing?
Keep the feature deferred until its consumer value and full lifecycle design
justify reconsidering the sequence-merging exclusion. A correct final sequence
alone would not establish correct inspection or disclosure behavior.
## 2. Required version declarations for selected sources ## 2. Required version declarations for selected sources

View File

@@ -19,6 +19,28 @@ func ExampleLoad() {
// Output: {true 3} <nil> // Output: {true 3} <nil>
} }
func ExampleLoad_sequenceDeclaration() {
type Item struct {
ID string `yaml:"id"`
}
type Config struct {
Items []Item `yaml:"items"`
}
result, err := configkit.Load[Config](configkit.Bytes("catalog.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
items:
- id: prompt
`)))
if err != nil {
fmt.Println(err)
return
}
fmt.Println(result.Config.Items[0].ID)
// Output: prompt
}
func ExampleNoSource() { func ExampleNoSource() {
type Config struct { type Config struct {
Workers int `yaml:"workers"` Workers int `yaml:"workers"`
@@ -154,8 +176,15 @@ func ExampleOverride() {
}) })
} }
result, err := configkit.Load(configkit.NoSource(), configkit.Options[Config]{ result, err := configkit.Load(configkit.NoSource(), configkit.Options[Config]{
Definition: configkit.Definition[Config]{Defaults: func() Config { return Config{Workers: 4} }}, Definition: configkit.Definition[Config]{
Overrides: overrides, Defaults: func() Config { return Config{Workers: 4} },
Resolve: func(configuration *Config, resolution *configkit.Resolution) error {
configkit.Default(resolution, configkit.Keys("workers"), &configuration.Workers, 100,
"preserve an explicitly supplied worker count")
return nil
},
},
Overrides: overrides,
}) })
fmt.Println(result.Config.Workers, result.Sources()[0].Label, err) fmt.Println(result.Config.Workers, result.Sources()[0].Label, err)
// Output: 0 --workers <nil> // Output: 0 --workers <nil>

View File

@@ -16,9 +16,10 @@ type PromptRecord struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Name string `yaml:"name"` Name string `yaml:"name"`
Messages []string `yaml:"messages"` Messages []string `yaml:"messages"`
Resource string `yaml:"resource"`
} }
// LoadPromptCatalog composes explicitly registered prompt records in the main // LoadPromptCatalog discovers explicitly registered prompt records in the main
// tier and replaces the lower-priority default collection as a whole. // tier and replaces the lower-priority default collection as a whole.
func LoadPromptCatalog(mainPath string) (*configkit.Result[PromptCatalogConfig], error) { func LoadPromptCatalog(mainPath string) (*configkit.Result[PromptCatalogConfig], error) {
return configkit.LoadLayered(configkit.Input{ return configkit.LoadLayered(configkit.Input{
@@ -28,16 +29,29 @@ func LoadPromptCatalog(mainPath string) (*configkit.Result[PromptCatalogConfig],
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: configkit.Bytes("default prompts", []byte("prompts:\n - id: default\n name: Default prompt\n messages: [Default message]\n")), Source: configkit.Bytes("default prompts", []byte("prompts:\n - id: default\n name: Default prompt\n messages: [Default message]\n")),
}}}}, }}}},
Definition: configkit.Definition[PromptCatalogConfig]{ ImportDefaults: configkit.ImportDefaults{
SequenceCompositions: []configkit.SequenceComposition{{ Pattern: []string{"curated*.yml", "generated-*.yml"},
Path: configkit.Keys("prompts"),
IdentityField: "id",
}},
Validate: validatePromptCatalog,
}, },
Definition: promptCatalogDefinition(),
}) })
} }
// LoadRequiredPromptCatalog keeps the common one-file import shape small.
func LoadRequiredPromptCatalog(mainPath string) (*configkit.Result[PromptCatalogConfig], error) {
return configkit.Load(configkit.File(mainPath), configkit.Options[PromptCatalogConfig]{
Imports: true, Definition: promptCatalogDefinition(),
})
}
func promptCatalogDefinition() configkit.Definition[PromptCatalogConfig] {
return configkit.Definition[PromptCatalogConfig]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
},
Validate: validatePromptCatalog,
}
}
func validatePromptCatalog(configuration *PromptCatalogConfig, validation *configkit.Validation) { func validatePromptCatalog(configuration *PromptCatalogConfig, validation *configkit.Validation) {
for index, prompt := range configuration.Prompts { for index, prompt := range configuration.Prompts {
if strings.TrimSpace(prompt.Name) == "" { if strings.TrimSpace(prompt.Name) == "" {

View File

@@ -3,6 +3,8 @@ package consumers_test
import ( import (
"errors" "errors"
"fmt" "fmt"
"io/fs"
pathpkg "path"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -20,8 +22,26 @@ func Example_identifiedPromptCatalog() {
for _, prompt := range result.Config.Prompts { for _, prompt := range result.Config.Prompts {
ids = append(ids, prompt.ID) ids = append(ids, prompt.ID)
} }
fmt.Println(ids, err) emptySearches := 0
// Output: [main curated weather welcome] <nil> for _, source := range result.Sources() {
for _, selection := range source.ImportSelections {
if selection.Selector == configkit.ImportSelectorDirectory && selection.Outcome == configkit.ImportSelectionNone {
emptySearches++
}
}
}
fmt.Println(ids, emptySearches, err)
// Output: [weather curated welcome main] 1 <nil>
}
func Example_requiredPromptCatalog() {
result, err := consumers.LoadRequiredPromptCatalog(fixture("prompt-catalog", "simple.yml"))
if err != nil {
fmt.Println(err)
return
}
fmt.Println(result.Config.Prompts[0].ID, err)
// Output: required <nil>
} }
func TestPromptCatalogImportsRegisteredItemsAndReplacesDefaults(t *testing.T) { func TestPromptCatalogImportsRegisteredItemsAndReplacesDefaults(t *testing.T) {
@@ -36,7 +56,7 @@ func TestPromptCatalogImportsRegisteredItemsAndReplacesDefaults(t *testing.T) {
if len(result.Config.Prompts) != 4 { if len(result.Config.Prompts) != 4 {
t.Fatalf("prompt records = %#v", result.Config.Prompts) t.Fatalf("prompt records = %#v", result.Config.Prompts)
} }
ids := []string{"main", "curated", "weather", "welcome"} ids := []string{"weather", "curated", "welcome", "main"}
for index, id := range ids { for index, id := range ids {
if result.Config.Prompts[index].ID != id { if result.Config.Prompts[index].ID != id {
t.Fatalf("prompt %d ID = %q, want %q", index, result.Config.Prompts[index].ID, id) t.Fatalf("prompt %d ID = %q, want %q", index, result.Config.Prompts[index].ID, id)
@@ -46,12 +66,15 @@ func TestPromptCatalogImportsRegisteredItemsAndReplacesDefaults(t *testing.T) {
if prompt.ID == "default" { if prompt.ID == "default" {
t.Fatalf("lower-priority default survived main sequence replacement: %#v", result.Config.Prompts) t.Fatalf("lower-priority default survived main sequence replacement: %#v", result.Config.Prompts)
} }
if prompt.ID == "ignored" {
t.Fatalf("excluded generated prompt was selected: %#v", result.Config.Prompts)
}
} }
var itemImports, helperImports int var itemImports, helperImports int
for _, source := range result.Sources() { for _, source := range result.Sources() {
switch filepath.Base(source.Path) { switch filepath.Base(source.Path) {
case "curated.yml", "generated-weather.yml", "generated-welcome.yml": case "curated.yml", "weather-feature.yml", "generated-welcome.yml":
if source.PlacementMode != configkit.PlacementModeSequenceItem || source.Destination.String() != "/prompts" { if source.PlacementMode != configkit.PlacementModeSequenceItem || source.Destination.String() != "/prompts" {
t.Fatalf("item source placement = %#v", source) t.Fatalf("item source placement = %#v", source)
} }
@@ -71,6 +94,36 @@ func TestPromptCatalogImportsRegisteredItemsAndReplacesDefaults(t *testing.T) {
} }
} }
func TestPromptCatalogMountsResourceFromScheduledPromptOrigin(t *testing.T) {
result, err := consumers.LoadPromptCatalog(fixture("prompt-catalog", "config.yml"))
if err != nil {
t.Fatal(err)
}
if result.Config.Prompts[2].ID != "welcome" || result.Config.Prompts[2].Resource != "welcome.txt" {
t.Fatalf("scheduled resource prompt = %#v", result.Config.Prompts)
}
resourcePath := configkit.Keys("prompts").Index(2).Key("resource")
info, ok := result.Metadata().Lookup(resourcePath)
if !ok || len(info.Origin.Locations) != 1 {
t.Fatalf("resource origin = %#v, %v", info, ok)
}
context, ok := result.SourceContext(info.Origin.Locations[0].SourceID)
if !ok || context.FS == nil || filepath.Base(context.BaseDir) != "prompts" {
t.Fatalf("resource source context = %#v, %v", context, ok)
}
contents, err := fs.ReadFile(context.FS, pathpkg.Join(context.BaseDir, result.Config.Prompts[2].Resource))
if err != nil || string(contents) != "Welcome resource text.\n" {
t.Fatalf("mounted resource = %q, %v", contents, err)
}
}
func TestRequiredPromptCatalogUsesOneRequiredFile(t *testing.T) {
result, err := consumers.LoadRequiredPromptCatalog(fixture("prompt-catalog", "simple.yml"))
if err != nil || len(result.Config.Prompts) != 1 || result.Config.Prompts[0].ID != "required" {
t.Fatalf("simple required prompt = %#v, %v", result, err)
}
}
func TestPromptCatalogLeavesDomainValidationWithConsumer(t *testing.T) { func TestPromptCatalogLeavesDomainValidationWithConsumer(t *testing.T) {
_, err := consumers.LoadPromptCatalog(fixture("prompt-catalog", "invalid.yml")) _, err := consumers.LoadPromptCatalog(fixture("prompt-catalog", "invalid.yml"))
var loadError *configkit.LoadError var loadError *configkit.LoadError

View File

@@ -0,0 +1,134 @@
package consumers
import (
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
// SequenceCatalogConfig is a source-backed prompt catalog with an application
// supplied run label and a value derived from that label during resolution.
type SequenceCatalogConfig struct {
RunLabel string `yaml:"run_label"`
ResolvedLabel string `yaml:"resolved_label"`
Prompts []SequenceCatalogEntry `yaml:"prompts"`
}
// SequenceCatalogEntry keeps prompt meaning, resources, and credentials in the
// consuming application while ConfigKit handles collection assembly.
type SequenceCatalogEntry struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Description string `yaml:"description"`
Messages []string `yaml:"messages"`
Resource string `yaml:"resource"`
Credential configkit.Secret `yaml:"credential"`
}
// LoadSequenceCatalog assembles imported main fragments over a file-backed
// underlay, applies one selected collection overlay, then resolves and validates
// the resulting application configuration.
func LoadSequenceCatalog(mainPath, overlayName, runLabel string) (*configkit.Result[SequenceCatalogConfig], error) {
root := filepath.Dir(mainPath)
filesystem := configkit.NewFilesystem("sequence catalog", os.DirFS(root))
filesystemContext := &configkit.FilesystemContext{BaseDir: ".", Root: ".", FS: &filesystem}
definition := configkit.Definition[SequenceCatalogConfig]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
},
Resolve: func(configuration *SequenceCatalogConfig, resolution *configkit.Resolution) error {
configuration.ResolvedLabel = "for:" + configuration.RunLabel
resolution.Record(configkit.Keys("resolved_label"), configkit.ChangeNormalization,
"derive the resolved label from the selected run label", configkit.Keys("run_label"))
return nil
},
Validate: validateSequenceCatalog,
}
return configkit.LoadLayered(configkit.Input{
Source: filesystem.File(filepath.Base(mainPath)),
Imports: true,
}, configkit.LayeredOptions[SequenceCatalogConfig]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: filesystem.File("underlay.yml"),
}}}},
Overlay: overlayName,
OverlayFilesystem: filesystemContext,
Overrides: []configkit.Override{{
Path: configkit.Keys("run_label"), Value: runLabel, Flag: "--run-label",
}},
Definition: definition,
})
}
// ReadSequenceCatalogResource follows the final prompt field's source context,
// including when record inheritance retained a resource from an underlay.
func ReadSequenceCatalogResource(result *configkit.Result[SequenceCatalogConfig], promptID string) (string, error) {
if result == nil {
return "", errors.New("sequence catalog result is nil")
}
for index, prompt := range result.Config.Prompts {
if prompt.ID != promptID {
continue
}
resourcePath := configkit.Keys("prompts").Index(index).Key("resource")
info, found := result.Metadata().Lookup(resourcePath)
if !found || len(info.Origin.Locations) != 1 {
return "", errors.New("prompt resource has no single declaring source")
}
context, found := result.SourceContext(info.Origin.Locations[0].SourceID)
if !found {
return "", errors.New("prompt resource source context is unavailable")
}
data, err := fs.ReadFile(context.FS, path.Join(context.BaseDir, prompt.Resource))
if err != nil {
return "", fmt.Errorf("read prompt resource: %w", err)
}
return string(data), nil
}
return "", fmt.Errorf("prompt %q is unavailable", promptID)
}
func validateSequenceCatalog(configuration *SequenceCatalogConfig, validation *configkit.Validation) {
for index, prompt := range configuration.Prompts {
if strings.TrimSpace(prompt.Name) == "" {
validation.Error(configkit.Keys("prompts").Index(index).Key("name"), "must not be empty")
}
if len(prompt.Messages) == 0 {
validation.Error(configkit.Keys("prompts").Index(index).Key("messages"), "must contain at least one message")
}
}
if len(configuration.Prompts) == 0 {
validation.Error(configkit.Keys("prompts"), "must contain at least one prompt")
}
}
// NamespacedMapCatalogConfig demonstrates typed application maps whose
// independent prompt sequences use root and dynamic namespace registrations.
type NamespacedMapCatalogConfig struct {
Prompts []map[string]any `yaml:"prompts"`
Collections map[string]map[string]any `yaml:"collections"`
}
// LoadNamespacedMapCatalog applies the same scalar identity constraint to one
// root collection and each namespace's independently scoped collection.
func LoadNamespacedMapCatalog(source configkit.Source) (*configkit.Result[NamespacedMapCatalogConfig], error) {
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}
return configkit.Load(source, configkit.Options[NamespacedMapCatalogConfig]{
Definition: configkit.Definition[NamespacedMapCatalogConfig]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
configkit.SequenceIdentityMatching(selector, "id"),
},
},
})
}

View File

@@ -0,0 +1,239 @@
package consumers_test
import (
"bytes"
"errors"
"fmt"
"os"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/ConfigKit/examples/consumers"
)
func Example_sequenceCatalog() {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
restoreOld, restoreNew := preserveEnvironment(oldSecret), preserveEnvironment(newSecret)
defer restoreOld()
defer restoreNew()
_ = os.Setenv(oldSecret, "disposable-old-example-credential")
_ = os.Setenv(newSecret, "disposable-new-example-credential")
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), "upsert", "preview")
if err != nil {
fmt.Println(err)
return
}
ids := make([]string, 0, len(result.Config.Prompts))
for _, prompt := range result.Config.Prompts {
ids = append(ids, prompt.ID)
}
shared := configkit.Keys("prompts").Index(1).Key("name")
info, _ := result.Metadata().Lookup(shared)
resource, resourceErr := consumers.ReadSequenceCatalogResource(result, "concise")
fmt.Println(ids, result.Config.ResolvedLabel, info.Explicit, len(result.Metadata().History(shared)), resource, resourceErr == nil, err)
// Output: [foundation shared welcome concise extra] for:preview true 2 Foundation resource text.
// true <nil>
}
func TestSequenceCatalogSelectedOverlayOperations(t *testing.T) {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
t.Setenv(oldSecret, "sequence-catalog-old-canary")
t.Setenv(newSecret, "sequence-catalog-new-canary")
tests := []struct {
name string
wantIDs []string
shared *consumers.SequenceCatalogEntry
}{
{name: "append", wantIDs: []string{"foundation", "shared", "welcome", "concise", "extra"}},
{name: "upsert", wantIDs: []string{"foundation", "shared", "welcome", "concise", "extra"}, shared: &consumers.SequenceCatalogEntry{ID: "shared", Name: "Shared replacement", Messages: []string{"Replacement message"}, Credential: configkit.SecretEnv(newSecret)}},
{name: "replace", wantIDs: []string{"replacement", "last"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), test.name, "test")
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("load selected overlay: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
gotIDs := make([]string, 0, len(result.Config.Prompts))
for _, prompt := range result.Config.Prompts {
gotIDs = append(gotIDs, prompt.ID)
}
if strings.Join(gotIDs, ",") != strings.Join(test.wantIDs, ",") {
t.Fatalf("prompt order = %v, want %v", gotIDs, test.wantIDs)
}
if test.shared == nil {
return
}
shared := result.Config.Prompts[1]
if shared.ID != test.shared.ID || shared.Name != test.shared.Name || len(shared.Messages) != 1 ||
shared.Messages[0] != test.shared.Messages[0] || shared.Description != "" || shared.Resource != "" || shared.Credential.EnvName() != newSecret {
t.Fatalf("upsert did not replace the complete matched record: %#v", shared)
}
})
}
}
func TestSequenceCatalogInspectionTracksResourcesAndSecretHistory(t *testing.T) {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
const oldCanary = "sequence-catalog-old-canary"
const newCanary = "sequence-catalog-new-canary"
t.Setenv(oldSecret, oldCanary)
t.Setenv(newSecret, newCanary)
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), "upsert", "history")
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("load sequence catalog: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
resourcePath := configkit.Keys("prompts").Index(3).Key("resource")
resourceInfo, found := result.Metadata().Lookup(resourcePath)
if !found || len(resourceInfo.Origin.Locations) != 1 || len(resourceInfo.Origin.Inheritance) != 1 {
t.Fatalf("inherited resource inspection = %#v, %v", resourceInfo, found)
}
resource, err := consumers.ReadSequenceCatalogResource(result, "concise")
if err != nil || resource != "Foundation resource text.\n" {
t.Fatalf("resource from inherited source context = %q, %v", resource, err)
}
if len(result.Metadata().History(configkit.Keys("resolved_label"))) == 0 {
t.Fatal("recorded CLI-driven resolution was absent from final inspection")
}
credentialPath := configkit.Keys("prompts").Index(1).Key("credential")
credentialHistory := result.Metadata().History(credentialPath)
if len(credentialHistory) == 0 {
t.Fatal("upsert did not retain the replaced secret reference in history")
}
var explanation bytes.Buffer
if err := result.WriteExplanation(&explanation, credentialPath); err != nil {
t.Fatal(err)
}
if strings.Contains(explanation.String(), oldCanary) || strings.Contains(explanation.String(), newCanary) {
t.Fatalf("secret explanation exposed a credential value: %s", explanation.String())
}
}
func TestNamespacedTypedMapIdentityScopesAreIndependent(t *testing.T) {
source := configkit.Bytes("namespaced maps", []byte(`
prompts:
- id: shared
text: root prompt
collections:
team:
prompts:
- id: shared
text: team prompt
operations:
prompts:
- id: shared
text: operations prompt
`))
result, err := consumers.LoadNamespacedMapCatalog(source)
if err != nil {
t.Fatal(err)
}
if result.Config.Prompts[0]["text"] != "root prompt" ||
result.Config.Collections["team"]["prompts"].([]any)[0].(map[string]any)["text"] != "team prompt" ||
result.Config.Collections["operations"]["prompts"].([]any)[0].(map[string]any)["text"] != "operations prompt" {
t.Fatalf("typed map collections = %#v", result.Config)
}
}
func TestNamespacedTypedMapIdentityFindsCollectionsIntroducedByInheritance(t *testing.T) {
result, err := consumers.LoadNamespacedMapCatalog(configkit.Bytes("inherited namespace", []byte(`
collections:
source:
prompts:
- id: shared
text: inherited prompt
copied:
extends: source
`)))
if err != nil {
t.Fatal(err)
}
copied := result.Config.Collections["copied"]["prompts"].([]any)
if len(copied) != 1 || copied[0].(map[string]any)["text"] != "inherited prompt" {
t.Fatalf("inherited typed-map collection = %#v", result.Config.Collections["copied"])
}
}
func Example_yamlDeclaredIdentityOnly() {
result, err := configkit.Load(configkit.Bytes("YAML identity only", []byte(`
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: initial
`)), configkit.Options[map[string]any]{
Overrides: []configkit.Override{{
Path: configkit.Keys("prompts"),
Value: []any{
map[string]any{"id": "same"},
map[string]any{"id": "same"},
},
Flag: "--prompts",
}},
})
fmt.Println(len(result.Config["prompts"].([]any)), err)
// Output: 2 <nil>
}
func TestYAMLDeclaredIdentityDoesNotRegisterFinalTypedConstraint(t *testing.T) {
result, err := configkit.Load(configkit.Bytes("YAML identity only", []byte(`
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: initial
`)), configkit.Options[map[string]any]{
Overrides: []configkit.Override{{
Path: configkit.Keys("prompts"),
Value: []any{
map[string]any{"id": "same"},
map[string]any{"id": "same"},
},
Flag: "--prompts",
}},
})
if err != nil {
t.Fatalf("YAML-only identity incorrectly became an application constraint: %v", err)
}
prompts := result.Config["prompts"].([]any)
if len(prompts) != 2 || prompts[0].(map[string]any)["id"] != prompts[1].(map[string]any)["id"] {
t.Fatalf("CLI result = %#v", result.Config)
}
}
func TestNamespacedTypedMapRejectsDuplicatesWithinOneCollection(t *testing.T) {
_, err := consumers.LoadNamespacedMapCatalog(configkit.Bytes("duplicate namespace", []byte(`
collections:
team:
prompts:
- id: duplicate
- id: duplicate
`)))
var loadError *configkit.LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 ||
loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict ||
loadError.Diagnostics()[0].Path.String() != "/collections/team/prompts" {
diagnostics := []configkit.Diagnostic(nil)
if loadError != nil {
diagnostics = loadError.Diagnostics()
}
t.Fatalf("duplicate namespaced identity error = %v (%#v)", err, diagnostics)
}
}

View File

@@ -1,11 +1,18 @@
configkit: configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
imports: imports:
- path: prompts/curated.yml - dir: prompts
exclude: [generated-ignored.yml]
destination: [prompts] destination: [prompts]
placement: sequence-item placement: sequence-item
- glob: prompts/generated-*.yml - dir: featured
pattern: [weather-*.yml]
destination: [prompts] destination: [prompts]
placement: sequence-item placement: sequence-item
- dir: optional
prompts: prompts:
- id: main - id: main
name: Main authored prompt name: Main authored prompt

View File

@@ -1,3 +1,8 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
prompts: prompts:
- id: invalid - id: invalid
name: Invalid prompt name: Invalid prompt

View File

@@ -0,0 +1 @@
This file does not match the application import defaults.

View File

@@ -1,6 +1,6 @@
configkit: configkit:
imports: imports:
- path: curated-id.yml - path: helpers/curated-id.yml
name: Curated summary name: Curated summary
messages: messages:
- Summarize the supplied content. - Summarize the supplied content.

View File

@@ -0,0 +1,3 @@
id: ignored
name: Excluded generated prompt
messages: [This record is excluded.]

View File

@@ -2,3 +2,4 @@ id: welcome
name: Welcome name: Welcome
messages: messages:
- Welcome the user. - Welcome the user.
resource: welcome.txt

View File

@@ -0,0 +1,3 @@
id: required
name: Required prompt
messages: [Use the required prompt.]

View File

@@ -0,0 +1 @@
Welcome resource text.

View File

@@ -0,0 +1,7 @@
configkit:
imports:
- path: prompts/required.yml
required: true
destination: [prompts]
placement: sequence-item
prompts: []

View File

@@ -0,0 +1,7 @@
run_label: source-default
configkit:
imports:
- dir: fragments
pattern: '*.yml'
destination: [prompts]
placement: sequence-item

View File

@@ -0,0 +1 @@
Foundation resource text.

View File

@@ -0,0 +1,4 @@
id: welcome
name: Welcome
messages: [Welcome message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,3 @@
id: concise
extends: {id: foundation}
name: Concise

View File

@@ -0,0 +1,10 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: append
prompts:
- id: extra
name: Extra
messages: [Extra message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,14 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
prompts:
- id: replacement
name: Replacement
messages: [Replacement message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET
- id: last
name: Last
messages: [Last message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,14 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: upsert
prompts:
- id: shared
name: Shared replacement
messages: [Replacement message]
credential: CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET
- id: extra
name: Extra
messages: [Extra message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,18 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: append
prompts:
- id: foundation
name: Foundation
description: Shared foundation instruction
messages: [Foundation message]
resource: foundation.txt
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET
- id: shared
name: Shared draft
description: This field must not survive a whole-record replacement
messages: [Draft message]
resource: shared.txt
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,249 @@
package configkit_test
import (
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFinalSequenceIdentityTreatsLateExtendsFieldsAsPayload(t *testing.T) {
for _, phase := range []string{"defaults", "resolution"} {
for _, component := range []any{"application-data", nil} {
for _, duplicate := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/%v/duplicate=%t", phase, component, duplicate), func(t *testing.T) {
items := []any{map[string]any{"extends": component}}
if duplicate {
items = append(items, map[string]any{"extends": component})
}
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "extends")},
Defaults: func() map[string]any {
if phase == "defaults" {
return map[string]any{"items": items}
}
return map[string]any{}
},
}
if phase == "resolution" {
definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["items"] = items
resolution.Record(configkit.Keys("items"), configkit.ChangeNormalization, "supply application payload")
return nil
}
}
loaded, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{Definition: definition})
if duplicate {
loadErr := requireFacadeLoadError(t, err)
diagnostics := loadErr.Diagnostics()
if loaded != nil || len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict || diagnostics[0].Path.String() != "/items" {
t.Fatalf("duplicate effective payload diagnostics = %#v", diagnostics)
}
return
}
if err != nil || loaded == nil {
t.Fatalf("ordinary effective identity was interpreted as a directive: %v", err)
}
value := loaded.Config["items"].([]any)[0].(map[string]any)
if actual, present := value["extends"]; !present || actual != component {
t.Fatalf("effective payload = %#v", value)
}
})
}
}
}
}
func TestFinalSequenceIdentityValidatesLateValues(t *testing.T) {
identity := configkit.SequenceIdentityAt(configkit.Keys("items"), "id")
for _, source := range []string{"defaults", "CLI", "resolution"} {
for _, shape := range []string{"duplicate", "missing", "non-scalar"} {
t.Run(source+"/"+shape, func(t *testing.T) {
items := finalIdentityRecords(shape)
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{identity},
Defaults: func() map[string]any {
if source == "defaults" {
return map[string]any{"items": items}
}
return map[string]any{}
},
}
options := configkit.Options[map[string]any]{Definition: definition}
switch source {
case "CLI":
options.Overrides = []configkit.Override{{Path: configkit.Keys("items"), Value: items, Flag: "--items"}}
case "resolution":
options.Definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["items"] = items
resolution.Record(configkit.Keys("items"), configkit.ChangeNormalization, "create effective records")
return nil
}
}
result, err := configkit.Load(configkit.NoSource(), options)
if result != nil {
t.Fatalf("invalid final identities returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) != 1 {
t.Fatalf("diagnostics = %#v", loadErr.Diagnostics())
}
diagnostic := loadErr.Diagnostics()[0]
wantCode, wantPath := configkit.CodeSequenceInvalid, "/items/0/id"
if shape == "duplicate" {
wantCode, wantPath = configkit.CodeSequenceConflict, "/items"
}
if diagnostic.Code != wantCode || diagnostic.Path.String() != wantPath {
t.Fatalf("diagnostic = %#v, want %s at %s", diagnostic, wantCode, wantPath)
}
if source == "CLI" && (diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-1") {
t.Fatalf("CLI identity diagnostic lost its source: %#v", diagnostic)
}
})
}
}
}
func TestFinalSequenceIdentityMatchesLateDynamicNamespaces(t *testing.T) {
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("items"),
}
for _, source := range []string{"defaults", "CLI", "resolution"} {
t.Run(source, func(t *testing.T) {
name := map[string]string{"defaults": "team", "CLI": "cloud", "resolution": "orbit"}[source]
path := configkit.Keys("collections", name, "items")
items := finalIdentityRecords("duplicate")
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(selector, "id"),
},
Defaults: func() map[string]any {
if source == "defaults" {
return map[string]any{"collections": map[string]any{name: map[string]any{"items": items}}}
}
return map[string]any{}
},
}
options := configkit.Options[map[string]any]{Definition: definition}
switch source {
case "CLI":
options.Overrides = []configkit.Override{{Path: path, Value: items, Flag: "--items"}}
case "resolution":
options.Definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["collections"] = map[string]any{name: map[string]any{"items": items}}
resolution.Record(configkit.Keys("collections"), configkit.ChangeNormalization, "create namespaced records")
return nil
}
}
result, err := configkit.Load(configkit.NoSource(), options)
if result != nil {
t.Fatalf("invalid dynamic identity returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict || diagnostics[0].Path.String() != path.String() {
t.Fatalf("dynamic identity diagnostics = %#v", diagnostics)
}
})
}
}
func TestFinalSequenceIdentityAppliesToLayeredLoads(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Defaults: func() map[string]any {
return map[string]any{"items": finalIdentityRecords("duplicate")}
},
}
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[map[string]any]{Definition: definition})
if result != nil {
t.Fatalf("invalid layered final identities returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("layered identity diagnostics = %#v", diagnostics)
}
}
func TestFinalSequenceIdentityUsesTypedNumbersAndIgnoresUnmatchedShapes(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("items"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("unmatched"), "id"),
},
}
_, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{
Definition: configkit.Definition[map[string]any]{
SequenceIdentities: definition.SequenceIdentities,
Defaults: func() map[string]any {
return map[string]any{"items": []any{
map[string]any{"id": int64(1)}, map[string]any{"id": float64(1)},
}}
},
},
})
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("typed numeric identity diagnostics = %#v", diagnostics)
}
valid, err := configkit.Load(configkit.Mapping("valid", map[string]any{
"items": []any{
map[string]any{"id": nil}, map[string]any{"id": true},
map[string]any{"id": int64(1)}, map[string]any{"id": "one"},
},
"other": []any{map[string]any{"id": "same"}, map[string]any{"id": "same"}},
}), configkit.Options[map[string]any]{Definition: definition})
if err != nil || valid == nil {
t.Fatalf("valid scalar identities and unmatched duplicate sequence = %#v, %v", valid, err)
}
if _, err := configkit.Load(configkit.Mapping("other-shapes", map[string]any{"items": nil}), configkit.Options[map[string]any]{Definition: definition}); err != nil {
t.Fatalf("null at a registered path was validated as an identity sequence: %v", err)
}
}
func TestFinalSequenceIdentityDiagnosticDoesNotRevealResolvedSecrets(t *testing.T) {
const canary = "FINAL-IDENTITY-CREDENTIAL-CANARY"
const secretName = "CONFIGKIT_FINAL_IDENTITY_SECRET"
t.Setenv(secretName, canary)
type item struct {
ID string `yaml:"id"`
Token configkit.Secret `yaml:"token"`
}
type config struct {
Items []item `yaml:"items"`
}
definition := configkit.Definition[config]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Defaults: func() config {
return config{Items: []item{
{ID: "duplicate", Token: configkit.SecretEnv(secretName)},
{ID: "duplicate", Token: configkit.SecretEnv(secretName)},
}}
},
}
result, err := configkit.Load(configkit.NoSource(), configkit.Options[config]{Definition: definition})
if result != nil {
t.Fatalf("duplicate identity returned a result: %#v", result.Config)
}
if err == nil || strings.Contains(err.Error(), canary) {
t.Fatalf("identity diagnostic leaked a resolved secret: %v", err)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("secret-bearing identity diagnostics = %#v", diagnostics)
}
}
func finalIdentityRecords(shape string) []any {
switch shape {
case "duplicate":
return []any{map[string]any{"id": "same"}, map[string]any{"id": "same"}}
case "missing":
return []any{map[string]any{"other": "value"}}
default:
return []any{map[string]any{"id": map[string]any{"nested": "value"}}}
}
}

View File

@@ -0,0 +1,99 @@
package configkit_test
import (
"errors"
"io/fs"
"os"
"path"
"path/filepath"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeNativeDiscoveryPreservesAliasIdentity(t *testing.T) {
root := t.TempDir()
for name, contents := range map[string]string{
"data/item.yml": "configkit:\n imports:\n - path: helper.yml\nvalue: item\n",
"data/helper.yml": "owner: target\nresource: resource.txt\n",
"aliases/helper.yml": "owner: alias\nresource: resource.txt\n",
"data/resource.txt": "target resource",
"aliases/resource.txt": "alias resource",
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}
for name, target := range map[string]string{"aliases/item.yml": "../data/item.yml", "linked": "data"} {
if err := os.Symlink(target, filepath.Join(root, filepath.FromSlash(name))); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
}
filesystem := configkit.NewFilesystem("aliases", os.DirFS(root))
type configuration struct {
Value string `yaml:"value"`
Owner string `yaml:"owner"`
Resource string `yaml:"resource"`
}
for _, test := range []struct {
name, declaration, item, helper, owner, resource string
}{
{"file/path", "path: aliases/item.yml", "aliases/item.yml", "aliases/helper.yml", "alias", "alias resource"},
{"file/dir", "dir: aliases\n pattern: item.yml", "aliases/item.yml", "aliases/helper.yml", "alias", "alias resource"},
{"file/glob", "glob: 'aliases/ite*.yml'", "aliases/item.yml", "aliases/helper.yml", "alias", "alias resource"},
{"directory/path", "path: linked/item.yml", "linked/item.yml", "linked/helper.yml", "target", "target resource"},
{"directory/dir", "dir: linked\n pattern: item.yml", "linked/item.yml", "linked/helper.yml", "target", "target resource"},
{"directory/glob", "glob: 'lin*/ite*.yml'", "linked/item.yml", "linked/helper.yml", "target", "target resource"},
} {
t.Run(test.name, func(t *testing.T) {
result, err := configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+test.declaration+"\n")), configkit.Options[configuration]{
Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
})
if err != nil {
t.Fatal(err)
}
if result.Config.Owner != test.owner || result.Config.Value != "item" {
t.Fatalf("configuration = %#v", result.Config)
}
sources := result.Sources()
if len(sources) != 3 || sources[1].Path != test.item || sources[2].Path != test.helper {
t.Fatalf("source identities = %#v", sources)
}
for field, source := range map[string]configkit.SourceInfo{"value": sources[1], "resource": sources[2]} {
info, ok := result.Metadata().Lookup(configkit.Keys(field))
if !ok || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != source.ID {
t.Fatalf("%s origin = %#v", field, info)
}
context, ok := result.SourceContext(source.ID)
if !ok || context.BaseDir != path.Dir(source.Path) {
t.Fatalf("%s context = %#v", field, context)
}
contents, err := fs.ReadFile(context.FS, path.Join(context.BaseDir, result.Config.Resource))
if err != nil || string(contents) != test.resource {
t.Fatalf("%s resource = %q, %v", field, contents, err)
}
}
})
}
// The same native path is still a repeat when separately included. An alias
// to a leaf file remains a different native namespace path, as for path imports.
for _, selector := range []string{"path: aliases/item.yml", "dir: aliases\n pattern: item.yml", "glob: 'aliases/ite*.yml'"} {
_, err := configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+selector+"\n - path: aliases/item.yml\n")), configkit.Options[configuration]{
Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != configkit.CodeImportRepeated {
t.Fatalf("repeated source: %v", err)
}
_, err = configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+selector+"\n - path: data/item.yml\n")), configkit.Options[configuration]{
Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
})
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict {
t.Fatalf("distinct native alias paths should reach composition: %v", err)
}
}
}

View File

@@ -0,0 +1,260 @@
package configkit
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/engine"
)
func TestImportDepthChecksSuffixAfterSaturation(t *testing.T) {
for _, test := range []struct {
depth string
valid bool
}{
{"999999999999999999999999.5", false},
{"999999999999999999999999e1", false},
{"999999999999999999999999", true},
{"-0", true},
} {
t.Run(test.depth, func(t *testing.T) {
main := writeFacadeImportFile(t, t.TempDir(), "main.yml", "value: true\nconfigkit:\n imports:\n - dir: absent\n recursive: true\n depth: "+test.depth+"\n")
result, err := Load[map[string]any](File(main), Options[map[string]any]{Imports: true})
if test.valid {
if err != nil || result.Config["value"] != true {
t.Fatalf("valid depth: %v", err)
}
return
}
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 {
t.Fatalf("invalid depth accepted: %v", err)
}
if diagnostic := loadError.Diagnostics()[0]; diagnostic.Code != CodeImportInvalid || diagnostic.Path.String() != "/configkit/imports/0/depth" {
t.Fatalf("invalid depth diagnostic = %#v", diagnostic)
}
})
}
}
func TestImportDefaultsApplyToTransitiveAndUnderlayImports(t *testing.T) {
type config struct {
Main string `yaml:"main"`
Transitive string `yaml:"transitive"`
Exception string `yaml:"exception"`
Underlay string `yaml:"underlay"`
IgnoredDeep string `yaml:"ignored_deep"`
}
directory := t.TempDir()
mainPath := writeFacadeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: parent/parent.yml
main: main
`)
writeFacadeImportFile(t, directory, "parent/parent.yml", `configkit:
imports:
- dir: generated
- dir: explicit
pattern: '*.yaml'
recursive: false
`)
transitivePath := writeFacadeImportFile(t, directory, "parent/generated/deep/transitive.cfg", "transitive: selected\n")
writeFacadeImportFile(t, directory, "parent/generated/ignored.yaml", "ignored_deep: wrong-pattern\n")
explicitPath := writeFacadeImportFile(t, directory, "parent/explicit/exception.yaml", "exception: declaration\n")
writeFacadeImportFile(t, directory, "parent/explicit/deep/ignored.yaml", "ignored_deep: wrong-depth\n")
writeFacadeImportFile(t, directory, "parent/explicit/ignored.cfg", "ignored_deep: wrong-explicit-pattern\n")
underlayPath := writeFacadeImportFile(t, directory, "underlay.yml", `configkit:
imports:
- dir: underlay-files
`)
underlayImportPath := writeFacadeImportFile(t, directory, "underlay-files/underlay.cfg", "underlay: selected\n")
recursive := true
loaded, err := LoadLayered(Input{Source: File(mainPath), Imports: true}, LayeredOptions[config]{
ImportDefaults: ImportDefaults{Recursive: &recursive, Pattern: []string{"*.cfg"}},
Underlays: []Underlay{{Inputs: []Input{{Source: File(underlayPath), Imports: true}}}},
})
if err != nil {
t.Fatalf("LoadLayered: %v", err)
}
if loaded.Config.Main != "main" || loaded.Config.Transitive != "selected" || loaded.Config.Exception != "declaration" || loaded.Config.Underlay != "selected" || loaded.Config.IgnoredDeep != "" {
t.Fatalf("configuration = %#v", loaded.Config)
}
paths := make(map[string]bool)
for _, source := range loaded.Sources() {
paths[source.Path] = true
}
for _, selected := range []string{transitivePath, explicitPath, underlayImportPath} {
if !paths[canonicalFacadeImportPath(t, selected)] {
t.Errorf("selected source %q missing from source records", selected)
}
}
}
func TestImportDefaultsPreserveEmptyPatternAndRequiredness(t *testing.T) {
type config struct {
Value string `yaml:"value"`
}
directory := t.TempDir()
mainPath := writeFacadeImportFile(t, directory, "main.yml", `configkit:
imports:
- dir: snippets
`)
writeFacadeImportFile(t, directory, "snippets/value.cfg", "value: imported\n")
unrestricted, err := Load(File(mainPath), Options[config]{Imports: true})
if err != nil || unrestricted.Config.Value != "imported" {
t.Fatalf("unspecified pattern = %#v, %v", unrestricted, err)
}
empty, err := Load(File(mainPath), Options[config]{Imports: true, ImportDefaults: ImportDefaults{Pattern: []string{}}})
if err != nil || empty.Config.Value != "" {
t.Fatalf("empty pattern = %#v, %v", empty, err)
}
emptySources := empty.Sources()
if len(emptySources) != 1 || len(emptySources[0].ImportSelections) != 1 || emptySources[0].ImportSelections[0].Path.String() != "/configkit/imports/0" || emptySources[0].ImportSelections[0].Selector != ImportSelectorDirectory || emptySources[0].ImportSelections[0].Outcome != ImportSelectionNone {
t.Fatalf("empty-pattern evidence = %#v", emptySources)
}
required := true
_, err = Load(File(mainPath), Options[config]{Imports: true, ImportDefaults: ImportDefaults{Required: &required, Pattern: []string{}}})
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeImportMissing {
t.Fatalf("required empty selection error = %T %v", err, err)
}
if sources := loadError.Sources(); len(sources) != 1 || len(sources[0].ImportSelections) != 0 {
t.Fatalf("required empty selection was recorded as optional: %#v", sources)
}
recursiveMain := writeFacadeImportFile(t, directory, "recursive.yml", `configkit:
imports:
- dir: recursive-files
`)
writeFacadeImportFile(t, directory, "recursive-files/nested/value.cfg", "value: selected\n")
recursive := true
patterns := []string{"*.cfg"}
engineOptions := toEngineOptions(Options[config]{Imports: true, ImportDefaults: ImportDefaults{Recursive: &recursive, Pattern: patterns}})
recursive = false
patterns[0] = "["
loadedEngine, err := engine.Load(File(recursiveMain).toEngine(), secretBinding(), engineOptions)
if err != nil || loadedEngine.Config.Value != "selected" {
t.Fatalf("load after mutating caller defaults = %#v, %v", loadedEngine, err)
}
}
func TestInvalidImportDefaultsPreflightBeforeOrdinaryAndLayeredReads(t *testing.T) {
type config struct {
Value string `yaml:"value"`
}
for _, layered := range []bool{false, true} {
input := strings.NewReader("value: read\n")
underlayInput := strings.NewReader("value: underlay\n")
defaultsCalled := false
resolveCalled := false
definition := Definition[config]{
Defaults: func() config { defaultsCalled = true; return config{} },
Resolve: func(*config, *Resolution) error { resolveCalled = true; return nil },
}
defaults := ImportDefaults{Pattern: []string{"["}}
var err error
if layered {
_, err = LoadLayered(Input{Source: Reader("reader.yml", input)}, LayeredOptions[config]{
Definition: definition, ImportDefaults: defaults,
Underlays: []Underlay{{Inputs: []Input{{Source: Reader("underlay.yml", underlayInput)}}}},
})
} else {
_, err = Load(Reader("reader.yml", input), Options[config]{Definition: definition, ImportDefaults: defaults})
}
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeOptionsInvalid {
if loadError != nil {
t.Fatalf("layered=%v diagnostics = %#v", layered, loadError.Diagnostics())
}
t.Fatalf("layered=%v error = %T %v", layered, err, err)
}
if input.Len() != len("value: read\n") || underlayInput.Len() != len("value: underlay\n") || defaultsCalled || resolveCalled {
t.Fatalf("layered=%v consumed input or called application hooks: remaining=%d underlay=%d defaults=%v resolve=%v", layered, input.Len(), underlayInput.Len(), defaultsCalled, resolveCalled)
}
}
}
func TestPublicImportSelectionSnapshotsAreDetachedAcrossResultErrorAndResolution(t *testing.T) {
type config struct {
Value string `yaml:"value"`
}
directory := t.TempDir()
mainPath := writeFacadeImportFile(t, directory, "main.yml", `configkit:
imports:
- dir: absent
value: loaded
`)
callbackPath := ""
loaded, err := Load(File(mainPath), Options[config]{Imports: true, Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error {
first, ok := resolution.Source("source-1")
if !ok || len(first.ImportSelections) != 1 {
t.Fatalf("resolution source = %#v, %v", first, ok)
}
first.ImportSelections[0].Path = Keys("callback mutation")
second, ok := resolution.Source("source-1")
if !ok || second.ImportSelections[0].Path.String() != "/configkit/imports/0" {
t.Fatalf("resolution snapshot changed after mutation: %#v", second)
}
callbackPath = second.ImportSelections[0].Path.String()
return nil
}}})
if err != nil {
t.Fatal(err)
}
if callbackPath != "/configkit/imports/0" {
t.Fatalf("callback path = %q", callbackPath)
}
sources := loaded.Sources()
if len(sources) != 1 || len(sources[0].ImportSelections) != 1 || sources[0].ImportSelections[0].Location.SourceID != sources[0].ID || sources[0].ImportSelections[0].Selector != ImportSelectorDirectory {
t.Fatalf("result source evidence = %#v", sources)
}
sources[0].ImportSelections[0].Path = Keys("result mutation")
if got := loaded.Sources()[0].ImportSelections[0].Path.String(); got != "/configkit/imports/0" {
t.Fatalf("result snapshot changed after mutation: %q", got)
}
failurePath := writeFacadeImportFile(t, directory, "failure.yml", `configkit:
imports:
- dir: absent
- path: required-missing.yml
required: true
`)
_, err = Load(File(failurePath), Options[config]{Imports: true})
var loadError *LoadError
if !errors.As(err, &loadError) {
t.Fatalf("failed load error = %T %v", err, err)
}
errorSources := loadError.Sources()
if len(errorSources) != 1 || len(errorSources[0].ImportSelections) != 1 || errorSources[0].ImportSelections[0].Path.String() != "/configkit/imports/0" {
t.Fatalf("error source evidence = %#v", errorSources)
}
errorSources[0].ImportSelections[0].Path = Keys("error mutation")
if got := loadError.Sources()[0].ImportSelections[0].Path.String(); got != "/configkit/imports/0" {
t.Fatalf("error snapshot changed after mutation: %q", got)
}
}
func writeFacadeImportFile(t *testing.T, root, name, contents string) string {
t.Helper()
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
return filename
}
func canonicalFacadeImportPath(t *testing.T, filename string) string {
t.Helper()
canonical, err := filepath.EvalSymlinks(filename)
if err != nil {
t.Fatal(err)
}
return filepath.ToSlash(canonical)
}

View File

@@ -0,0 +1,135 @@
package configkit
import (
"errors"
"io/fs"
"path"
"strings"
"testing"
"testing/fstest"
)
func TestDiscoveredItemHelpersPreserveNestedInheritedResourceEvidence(t *testing.T) {
const canary = "PRIVATE-CONSUMED-HEADER-CANARY"
filesystem := NewFilesystem("bundle", fstest.MapFS{
"bundle/config/main.yml": {Data: []byte(`configkit:
imports:
- dir: records
pattern: '*-item.yml'
destination: [services, records]
placement: sequence-item
services:
records:
- id: authored
version: authored
`)},
"bundle/config/records/a-item.yml": {Data: []byte(`configkit:
imports:
- path: helper.yml
version: ` + canary + `
entries:
parent:
resource: icon.txt
child:
extends: parent
`)},
"bundle/config/records/helper.yml": {Data: []byte("id: helper-supplied\n")},
"bundle/config/records/icon.txt": {Data: []byte("icon payload")},
})
result, err := Load(filesystem.File("bundle/config/main.yml"), Options[map[string]any]{
Imports: true,
Filesystem: &FilesystemContext{Root: "bundle", FS: &filesystem},
Definition: Definition[map[string]any]{
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceIdentities: []SequenceIdentity{SequenceIdentityAt(Keys("services", "records"), "id")},
Secret: func(candidate Path) bool {
return candidate.String() == "/services/records/0/version"
},
},
SourceVersionMode: SourceVersionModeIfPresent,
})
if err != nil {
var loadError *LoadError
if errors.As(err, &loadError) {
t.Fatalf("load discovered item: %v; diagnostics: %#v", err, loadError.Diagnostics())
}
t.Fatalf("load discovered item: %v", err)
}
services := result.Config["services"].(map[string]any)
records := services["records"].([]any)
if len(records) != 2 || records[0].(map[string]any)["id"] != "helper-supplied" || records[0].(map[string]any)["version"] != nil || records[1].(map[string]any)["version"] != "authored" {
t.Fatalf("nested records = %#v", records)
}
childPath := Keys("services", "records").Index(0).Key("entries").Key("child").Key("resource")
child := records[0].(map[string]any)["entries"].(map[string]any)["child"].(map[string]any)
if child["resource"] != "icon.txt" {
t.Fatalf("inherited relative resource = %#v", child)
}
sources := result.Sources()
filesByID := make(map[string]string, len(sources))
for _, source := range sources {
filesByID[source.ID] = source.Path
}
var itemSource SourceInfo
for _, source := range sources {
if source.Path == "bundle/config/records/a-item.yml" {
itemSource = source
}
}
if itemSource.VersionCheck == nil || !itemSource.VersionCheck.Version.Present || !itemSource.VersionCheck.Redacted || itemSource.VersionCheck.Version.Text != "" {
t.Fatalf("consumed header inspection = %#v", itemSource.VersionCheck)
}
idInfo, ok := result.Metadata().Lookup(Keys("services", "records").Index(0).Key("id"))
if !ok || len(idInfo.Origin.Locations) != 1 || filesByID[idInfo.Origin.Locations[0].SourceID] != "bundle/config/records/helper.yml" {
t.Fatalf("assigned helper ID origin = %#v", idInfo)
}
resourceInfo, ok := result.Metadata().Lookup(childPath)
if !ok || len(resourceInfo.Origin.Locations) != 1 || filesByID[resourceInfo.Origin.Locations[0].SourceID] != "bundle/config/records/a-item.yml" {
t.Fatalf("inherited resource metadata = %#v", resourceInfo)
}
explanation, ok := result.Explain(childPath)
if !ok || explanation.Effective.YAML != "icon.txt\n" || len(explanation.Origin.Locations) != 1 || explanation.Origin.Locations[0].SourceID != resourceInfo.Origin.Locations[0].SourceID {
t.Fatalf("inherited resource explanation = %#v", explanation)
}
context, ok := result.SourceContext(resourceInfo.Origin.Locations[0].SourceID)
if !ok || context.FS == nil || context.BaseDir != "config/records" {
t.Fatalf("inherited resource context = %#v, %v", context, ok)
}
contents, err := fs.ReadFile(context.FS, path.Join(context.BaseDir, child["resource"].(string)))
if err != nil || string(contents) != "icon payload" {
t.Fatalf("inherited resource read = %q, %v", contents, err)
}
_, err = Load(filesystem.File("bundle/config/main.yml"), Options[map[string]any]{
Imports: true,
Filesystem: &FilesystemContext{Root: "bundle", FS: &filesystem},
Definition: Definition[map[string]any]{
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceIdentities: []SequenceIdentity{SequenceIdentityAt(Keys("services", "records"), "version")},
Secret: func(candidate Path) bool {
return candidate.String() == "/services/records/0/version"
},
},
SourceVersionMode: SourceVersionModeIfPresent,
})
loadError, ok := err.(*LoadError)
if !ok || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeComposeInvalid {
t.Fatalf("consumed header supplied an identity: %v", err)
}
if strings.Contains(err.Error(), canary) {
t.Fatalf("diagnostic exposed the consumed secret header: %v", err)
}
var failedItemSource SourceInfo
foundFailedItem := false
for _, source := range loadError.Sources() {
if source.Path == "bundle/config/records/a-item.yml" {
failedItemSource = source
foundFailedItem = true
break
}
}
if !foundFailedItem || failedItemSource.VersionCheck == nil || failedItemSource.VersionCheck.Version.Text != "" || !failedItemSource.VersionCheck.Redacted {
t.Fatalf("failed-load source inspection exposed or omitted header evidence: %#v", failedItemSource)
}
}

119
facade_import_root_test.go Normal file
View File

@@ -0,0 +1,119 @@
package configkit_test
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeDirectoryRootBelowContainedAbsoluteSymlink(t *testing.T) {
root, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
parts := filepath.Join(root, "real", "parts")
if err := os.MkdirAll(parts, 0o700); err != nil {
t.Fatal(err)
}
partPath := filepath.Join(parts, "part.yml")
if err := os.WriteFile(partPath, []byte("value: included\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(root, "real"), filepath.Join(root, "absolute")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
for _, follow := range []bool{true, false} {
t.Run(fmt.Sprintf("follow=%t", follow), func(t *testing.T) {
mainPath := filepath.Join(root, "main.yml")
contents := fmt.Sprintf("configkit:\n imports:\n - dir: absolute/parts\n pattern: '*.yml'\n follow: %t\n", follow)
if err := os.WriteFile(mainPath, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
result, err := configkit.Load[map[string]any](configkit.File(mainPath), configkit.Options[map[string]any]{Imports: true})
if err != nil {
t.Fatal(err)
}
if result.Config["value"] != "included" {
t.Fatalf("configuration = %#v", result.Config)
}
if sources := result.Sources(); len(sources) != 2 || sources[1].Path != partPath {
t.Fatalf("canonical imported source = %#v", sources)
}
})
}
}
func TestFacadeNativeDiscoveryWithNestedRoot(t *testing.T) {
root := t.TempDir()
parts := filepath.Join(root, "nested/root/parts")
if err := os.MkdirAll(parts, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(parts, "value.yml"), []byte("value: 1\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("parts", filepath.Join(root, "nested/root/linked")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
filesystem := configkit.NewFilesystem("nested", os.DirFS(root))
context := &configkit.FilesystemContext{FS: &filesystem, Root: "nested/root", BaseDir: "nested/root"}
for _, selector := range []string{
"dir: parts", "glob: 'parts/*.yml'", "dir: linked", "glob: 'lin*/*.yml'",
} {
t.Run(selector, func(t *testing.T) {
result, err := configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+selector+"\n")), configkit.Options[map[string]any]{Imports: true, Filesystem: context})
if err != nil {
t.Fatal(err)
}
if result.Config["value"] != int64(1) {
t.Fatalf("configuration = %#v", result.Config)
}
})
}
if err := os.WriteFile(filepath.Join(root, "nested/outside.yml"), []byte("value: outside\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("../../outside.yml", filepath.Join(parts, "escape.yml")); err != nil {
t.Fatal(err)
}
for _, selector := range []string{"dir: parts", "glob: 'parts/*.yml'"} {
_, err := configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+selector+"\n")), configkit.Options[map[string]any]{Imports: true, Filesystem: context})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != configkit.CodeImportOutsideRoot {
t.Fatalf("escaping %s: %v", selector, err)
}
}
}
func TestFacadeDiscoveryAllowsFiniteLinkReuse(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "parts"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "parts/value.yml"), []byte("value: 1\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("..", filepath.Join(root, "parts/up")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
filesystem := configkit.NewFilesystem("finite", os.DirFS(root))
for _, native := range []bool{false, true} {
for _, selector := range []string{"dir: parts/up/parts/up/parts\n pattern: '*.yml'", "glob: 'parts/up/parts/up/parts/*.yml'"} {
context := &configkit.FilesystemContext{Root: root, BaseDir: root}
if native {
context = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
}
result, err := configkit.Load(configkit.Bytes("main", []byte("configkit:\n imports:\n - "+selector+"\n")), configkit.Options[map[string]any]{Imports: true, Filesystem: context})
if err != nil {
t.Fatalf("native=%v selector=%s: %v", native, selector, err)
}
if result.Config["value"] != int64(1) {
t.Fatalf("configuration = %#v", result.Config)
}
}
}
}

View File

@@ -91,6 +91,214 @@ func TestFacadeInheritancePreservesPresenceOriginsAndHistory(t *testing.T) {
} }
} }
func TestFacadeIdentifiedInheritanceResolvesChainsAndNestedCollections(t *testing.T) {
const secretName = "CONFIGKIT_IDENTIFIED_INHERITANCE_SECRET"
const canary = "IDENTIFIED-INHERITANCE-SECRET-CANARY"
t.Setenv(secretName, canary)
type step struct {
ID string `yaml:"id"`
Value string `yaml:"value"`
}
type job struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Credential configkit.Secret `yaml:"credential"`
Resource string `yaml:"resource"`
Steps []step `yaml:"steps"`
Settings map[string]map[string]string `yaml:"settings"`
}
type configuration struct {
Jobs []job `yaml:"jobs"`
}
input := []byte(`jobs:
- id: child
extends: {id: parent}
name: child-name
- id: parent
extends: {id: base}
- id: base
name: base-name
credential: CONFIGKIT_IDENTIFIED_INHERITANCE_SECRET
resource: icon.txt
steps:
- id: base-step
value: base-step-value
- id: second-step
value: second-step-value
settings:
primary: {value: base-setting}
secondary: {extends: primary}
`)
filesystem := configkit.NewFilesystem("identified-bundle", fstest.MapFS{
"config/identified.yml": &fstest.MapFile{Data: input},
"config/icon.txt": &fstest.MapFile{Data: []byte("identified-parent-resource")},
})
registrations := []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("jobs"), "id"),
}
result, err := configkit.Load(filesystem.File("config/identified.yml"), configkit.Options[configuration]{
Definition: configkit.Definition[configuration]{
SequenceIdentities: registrations,
Secret: func(path configkit.Path) bool {
return path.String() == "/jobs/0/credential" || path.String() == "/jobs/1/credential" || path.String() == "/jobs/2/credential"
},
},
})
if err != nil {
if loadError, ok := err.(*configkit.LoadError); ok {
t.Fatalf("load identified inheritance: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
if len(result.Config.Jobs) != 3 || result.Config.Jobs[0].ID != "child" || result.Config.Jobs[1].ID != "parent" || result.Config.Jobs[2].ID != "base" {
t.Fatalf("inherited job positions and identities = %#v", result.Config.Jobs)
}
child := result.Config.Jobs[0]
if child.Name != "child-name" || child.Credential.Value() != canary || child.Resource != "icon.txt" || len(child.Steps) != 2 {
t.Fatalf("inherited child = %#v", child)
}
if got := []string{child.Steps[0].ID, child.Steps[1].ID}; strings.Join(got, ",") != "base-step,second-step" {
t.Fatalf("inherited nested steps = %v", got)
}
if child.Steps[0].Value != "base-step-value" || child.Steps[1].Value != "second-step-value" || child.Settings["secondary"]["value"] != "base-setting" {
t.Fatalf("inherited nested collection and mapping = %#v", child)
}
credentialPath := configkit.Keys("jobs").Index(0).Key("credential")
credentialInfo, found := result.Metadata().Lookup(credentialPath)
if !found || len(credentialInfo.Origin.Inheritance) != 2 {
t.Fatalf("inherited secret provenance = %#v, %v", credentialInfo, found)
}
resourceInfo, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("resource"))
if !found || len(resourceInfo.Origin.Inheritance) != 2 || len(resourceInfo.Origin.Locations) != 1 {
t.Fatalf("inherited resource provenance = %#v, %v", resourceInfo, found)
}
context, found := result.SourceContext(resourceInfo.Origin.Locations[0].SourceID)
if !found || context.BaseDir != "config" || context.FS == nil {
t.Fatalf("inherited resource context = %#v, %v", context, found)
}
resource, readErr := fs.ReadFile(context.FS, context.BaseDir+"/"+child.Resource)
if readErr != nil || string(resource) != "identified-parent-resource" {
t.Fatalf("inherited resource = %q, %v", resource, readErr)
}
resourceHistory, found := result.Explain(configkit.Keys("jobs").Index(0).Key("resource"))
if !found || len(resourceHistory.History) == 0 {
t.Fatalf("inherited resource history = %#v, %v", resourceHistory, found)
}
credentialHistory, found := result.Explain(credentialPath)
if !found || !credentialHistory.Effective.Redacted {
t.Fatalf("inherited secret explanation = %#v, %v", credentialHistory, found)
}
identityInfo, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("id"))
if !found || !identityInfo.Explicit || len(identityInfo.Origin.Inheritance) != 0 ||
len(identityInfo.Origin.Locations) != 1 {
t.Fatalf("child identity origin and presence = %#v, %v", identityInfo, found)
}
if _, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("extends")); found {
t.Fatal("identified record relationship remained in effective metadata")
}
var rendered bytes.Buffer
if err := result.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), canary) || strings.Contains(rendered.String(), "extends:") {
t.Fatalf("effective inherited configuration = %q, %v", rendered.String(), err)
}
}
func TestFacadeIdentifiedInheritanceFailureDoesNotRevealSecretCanary(t *testing.T) {
const secretName = "CONFIGKIT_IDENTIFIED_INHERITANCE_FAILURE_SECRET"
const canary = "IDENTIFIED-INHERITANCE-FAILURE-CANARY"
t.Setenv(secretName, canary)
type job struct {
ID string `yaml:"id"`
Credential configkit.Secret `yaml:"credential"`
}
type configuration struct {
Jobs []job `yaml:"jobs"`
}
input := []byte(`jobs:
- id: child
extends: {id: missing}
credential: CONFIGKIT_IDENTIFIED_INHERITANCE_FAILURE_SECRET
`)
_, err := configkit.Load(configkit.Bytes("identified-failure.yml", input), configkit.Options[configuration]{
Definition: configkit.Definition[configuration]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("jobs"), "id")},
Secret: func(path configkit.Path) bool { return path.String() == "/jobs/0/credential" },
},
})
if err == nil || strings.Contains(err.Error(), canary) {
t.Fatalf("identified inheritance failure = %v; secret canary must remain undisclosed", err)
}
}
func TestFacadeLayeredIdentifiedInheritanceUsesSurvivingRecords(t *testing.T) {
policy := func(mode string) string {
return "configkit:\n sequences:\n - destination: [jobs]\n key: id\n mode: " + mode + "\n"
}
tests := []struct {
name string
lower string
higher string
wantCode configkit.Code
wantValue string
wantSuccess bool
}{
{
name: "upserted parent supplies its final payload",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: parent, value: higher}, {id: child, extends: {id: parent}}]\n",
wantValue: "higher", wantSuccess: true,
},
{
name: "replacement removes a lower parent before lookup",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("replace") + "jobs: [{id: child, extends: {id: parent}}]\n",
wantCode: configkit.CodeInheritanceMissing,
},
{
name: "upserted child without a directive replaces the invalid lower child",
lower: policy("upsert") + "jobs: [{id: child, extends: {id: missing}, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: child, value: higher}]\n",
wantValue: "higher", wantSuccess: true,
},
{
name: "self-reference cannot resolve through a lower predecessor",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: parent, extends: {id: parent}, value: higher}]\n",
wantCode: configkit.CodeInheritanceCycle,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("main.yml", []byte(test.higher))}, configkit.LayeredOptions[map[string]any]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower.yml", []byte(test.lower))}}}},
})
if test.wantSuccess {
if err != nil {
t.Fatal(err)
}
jobs, ok := result.Config["jobs"].([]any)
if !ok || len(jobs) == 0 {
t.Fatalf("assembled jobs = %#v", result.Config["jobs"])
}
child, ok := jobs[len(jobs)-1].(map[string]any)
if !ok || child["value"] != test.wantValue {
t.Fatalf("surviving record = %#v, want inherited value %q", jobs[len(jobs)-1], test.wantValue)
}
return
}
loadError, ok := err.(*configkit.LoadError)
if !ok {
t.Fatalf("load error = %v, want code %s", err, test.wantCode)
}
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Code == test.wantCode {
return
}
}
t.Fatalf("load diagnostics = %#v, want code %s", loadError.Diagnostics(), test.wantCode)
})
}
}
func TestFacadeInheritanceLayeredPrecedenceAndSourceContext(t *testing.T) { func TestFacadeInheritanceLayeredPrecedenceAndSourceContext(t *testing.T) {
filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{ filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{
"config/main.yml": &fstest.MapFile{Data: []byte("entries: {child: {extends: parent}}\n")}, "config/main.yml": &fstest.MapFile{Data: []byte("entries: {child: {extends: parent}}\n")},
@@ -206,27 +414,24 @@ func TestFacadeInheritanceFailureStopsLaterCallbacks(t *testing.T) {
} }
} }
func TestFacadeInheritanceResolvesAfterImportsAndCLIReferenceChange(t *testing.T) { func TestFacadeCLIOverridesCannotChangeInheritanceRelationship(t *testing.T) {
filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{ filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}}\n")}, "main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}}\n")},
"parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}, second: {count: 2}}\n")}, "parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}, second: {count: 2}}\n")},
}) })
result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{ _, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{
Imports: true, Overrides: []configkit.Override{{Path: configkit.Keys("entries", "child", "extends"), Value: "second", Flag: "--extends"}}, Imports: true, Overrides: []configkit.Override{{Path: configkit.Keys("entries", "child", "extends"), Value: "second", Flag: "--extends"}},
}) })
if err != nil || result.Config.Entries["child"].Count != 2 { loadError, ok := err.(*configkit.LoadError)
t.Fatalf("import and CLI reference = %#v, %v", result, err) if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeOverrideDirective {
} t.Fatalf("CLI inheritance relationship change = %v", err)
history := result.Metadata().History(configkit.Keys("entries", "child", "extends"))
if len(history) < 3 || history[len(history)-1].Origin.Operation != configkit.ChangeInheritance {
t.Fatalf("superseded reference history = %#v", history)
} }
conflicting := configkit.NewFilesystem("bundle", fstest.MapFS{ conflicting := configkit.NewFilesystem("bundle", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}, first: {count: 3}}\n")}, "main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}, first: {count: 3}}\n")},
"parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}}\n")}, "parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}}\n")},
}) })
_, err = configkit.Load[inheritedConfiguration](conflicting.File("main.yml"), configkit.Options[inheritedConfiguration]{Imports: true}) _, err = configkit.Load[inheritedConfiguration](conflicting.File("main.yml"), configkit.Options[inheritedConfiguration]{Imports: true})
loadError, ok := err.(*configkit.LoadError) loadError, ok = err.(*configkit.LoadError)
if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict { if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict {
t.Fatalf("strict conflict before expansion = %v", err) t.Fatalf("strict conflict before expansion = %v", err)
} }

View File

@@ -0,0 +1,69 @@
package configkit_test
import (
"fmt"
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeItemScopesDoNotReinterpretGlobalRegistrations(t *testing.T) {
t.Parallel()
for _, layered := range []bool{false, true} {
for _, dynamic := range []bool{false, true} {
for _, missingOuterID := range []bool{false, true} {
t.Run(fmt.Sprintf("layered=%t/dynamic=%t/missing-outer-id=%t", layered, dynamic, missingOuterID), func(t *testing.T) {
payload := "prompts: [{text: helper}]\n"
unused := configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id")
if dynamic {
payload = "catalog: {local: {prompts: [{text: helper}]}}\n"
unused = configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("catalog"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts"),
}, "id")
}
item := "configkit: {imports: [{path: helper.yml}, {path: child.yml, destination: [children], placement: sequence-item}]}\n"
if !missingOuterID {
item += "id: outer\n"
}
files := configkit.NewFilesystem("items", fstest.MapFS{
"main.yml": {Data: []byte("configkit: {imports: [{path: item.yml, destination: [items], placement: sequence-item}]}\n")},
"item.yml": {Data: []byte(item)}, "helper.yml": {Data: []byte(payload)},
"child.yml": {Data: []byte("id: child\nconfigkit: {imports: [{path: child-helper.yml}]}\n")},
"child-helper.yml": {Data: []byte(payload)},
})
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
unused, configkit.SequenceIdentityAt(configkit.Keys("items"), "id"),
}}
var result *configkit.Result[map[string]any]
var err error
if layered {
result, err = configkit.LoadLayered(configkit.Input{Source: files.File("main.yml"), Imports: true}, configkit.LayeredOptions[map[string]any]{Definition: definition})
} else {
result, err = configkit.Load(files.File("main.yml"), configkit.Options[map[string]any]{Imports: true, Definition: definition})
}
if missingOuterID {
failure := requireFacadeLoadError(t, err)
if result != nil || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != configkit.CodeComposeInvalid || failure.Diagnostics()[0].Path.String() != "/items/0/id" {
t.Fatalf("outer constraint diagnostics = %#v", failure.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
for _, record := range []configkit.Path{configkit.Keys("items").Index(0), configkit.Keys("items").Index(0).Key("children").Index(0)} {
path := record
if dynamic {
path = path.Key("catalog").Key("local")
}
info, found := result.Metadata().Lookup(path.Key("prompts").Index(0).Key("text"))
if !found || !info.Explicit {
t.Fatalf("nested helper payload lost at %s", path)
}
}
})
}
}
}
}

View File

@@ -0,0 +1,64 @@
package configkit_test
import (
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeItemDomainVersionsKeepEffectiveScope(t *testing.T) {
for _, test := range []struct {
name, item, child, helper string
underlay, mainItem bool
}{
{name: "item", item: "id: item\nversion: domain\n"},
{name: "helper", item: "id: item\nconfigkit: {imports: [{path: helper.yml}]}\n", helper: "version: domain\n"},
{name: "nested", item: "id: parent\nconfigkit: {imports: [{path: child.yml, destination: [children], placement: sequence-item}]}\n", child: "id: child\nversion: domain\n"},
{name: "underlay", item: "id: item\nversion: domain\n", underlay: true},
{name: "main-item", item: "id: parent\nconfigkit: {imports: [{path: child.yml, destination: [children], placement: sequence-item}]}\n", child: "id: child\nversion: domain\n", mainItem: true},
} {
t.Run(test.name, func(t *testing.T) {
files := fstest.MapFS{
"main.yml": {Data: []byte("version: 1\nconfigkit: {imports: [{path: item.yml, destination: [items], placement: sequence-item}]}\n")},
"item.yml": {Data: []byte(test.item)},
"child.yml": {Data: []byte(test.child)},
"helper.yml": {Data: []byte(test.helper)},
"underlay.yml": {Data: []byte("configkit: {imports: [{path: item.yml, destination: [items], placement: sequence-item}]}\n")},
}
filesystem := configkit.NewFilesystem("items", files)
definition := configkit.Definition[map[string]any]{Version: &configkit.VersionRule{
Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { return nil },
}}
var result *configkit.Result[map[string]any]
var err error
switch {
case test.underlay:
result, err = configkit.LoadLayered(configkit.Input{Source: configkit.Mapping("main", map[string]any{"version": "1"})}, configkit.LayeredOptions[map[string]any]{
Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: filesystem.File("underlay.yml"), Imports: true}}}},
})
case test.mainItem:
result, err = configkit.Load(filesystem.File("item.yml"), configkit.Options[map[string]any]{
Imports: true, Destination: configkit.Keys("items"), PlacementMode: configkit.PlacementModeSequenceItem, Definition: definition,
})
default:
result, err = configkit.Load(filesystem.File("main.yml"), configkit.Options[map[string]any]{Imports: true, Definition: definition})
}
if err != nil {
t.Fatal(err)
}
item := result.Config["items"].([]any)[0].(map[string]any)
fieldPath := configkit.Keys("items").Index(0).Key("version")
if test.child != "" {
item = item["children"].([]any)[0].(map[string]any)
fieldPath = configkit.Keys("items").Index(0).Key("children").Index(0).Key("version")
}
if item["version"] != "domain" {
t.Fatalf("domain version = %#v", item)
}
if info, ok := result.Metadata().Lookup(fieldPath); !ok || !info.Explicit {
t.Fatalf("version metadata = %#v, %v", info, ok)
}
})
}
}

View File

@@ -0,0 +1,240 @@
package configkit_test
import (
"errors"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeSequenceIdentitySnapshotsSelectorsAndFields(t *testing.T) {
fields := []string{"namespace", "id", "id"}
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}
identity := configkit.SequenceIdentityMatching(selector, fields...)
selector[1] = configkit.SequenceLiteralKey("other")
fields[0] = "changed"
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{identity}}
underlay := configkit.Mapping("underlay", map[string]any{"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"namespace": "team", "id": "kept"}}},
}})
main := configkit.Mapping("main", map[string]any{"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"namespace": "team", "id": "added"}}},
}})
loaded, err := configkit.LoadLayered(configkit.Input{Source: main}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}}}},
})
if err != nil {
t.Fatalf("load after mutating constructor inputs: %v", err)
}
prompts := loaded.Config["collections"].(map[string]any)["team"].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Fatalf("registered compound identity did not append records: %#v", prompts)
}
}
func TestFacadeSequenceIdentityUsesExactAndDynamicSelectors(t *testing.T) {
dynamic := configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id")
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "namespace", "id"),
dynamic,
}}
underlay := configkit.Mapping("underlay", map[string]any{
"prompts": []any{map[string]any{"namespace": "alpha", "id": "same"}},
"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-one"}}},
"cloud": map[string]any{"prompts": []any{map[string]any{"id": "cloud-one"}}},
},
})
main := configkit.Mapping("main", map[string]any{
"prompts": []any{map[string]any{"namespace": "beta", "id": "same"}},
"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-two"}}},
"cloud": map[string]any{"prompts": []any{map[string]any{"id": "cloud-two"}}},
},
})
loaded, err := configkit.LoadLayered(configkit.Input{Source: main}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}}}},
})
if err != nil {
t.Fatalf("load mixed exact and dynamic registrations: %v", err)
}
rootRecords := loaded.Config["prompts"].([]any)
if len(rootRecords) != 2 {
t.Fatalf("compound root identities = %#v", rootRecords)
}
collections := loaded.Config["collections"].(map[string]any)
for _, name := range []string{"team", "cloud"} {
prompts := collections[name].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Errorf("dynamic prompts for %q = %#v", name, prompts)
}
}
}
func TestFacadeSequenceIdentityLiteralStarIsNotWildcard(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceLiteralKey("*"),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
underlay := configkit.Mapping("underlay", map[string]any{"collections": map[string]any{
"*": map[string]any{"prompts": []any{map[string]any{"id": "one"}}},
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-one"}}},
}})
main := configkit.Mapping("main", map[string]any{"collections": map[string]any{
"*": map[string]any{"prompts": []any{map[string]any{"id": "two"}}},
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-two"}}},
}})
_, err := configkit.LoadLayered(configkit.Input{Source: configkit.Mapping("main", map[string]any{})}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}, {Source: main}}}},
})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Path.String() != "/collections/team/prompts" {
t.Fatalf("literal star registration also matched another key: %#v", loadErr.Diagnostics())
}
wildcard := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
loaded, err := configkit.LoadLayered(configkit.Input{Source: configkit.Mapping("main", map[string]any{})}, configkit.LayeredOptions[map[string]any]{
Definition: wildcard,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}, {Source: main}}}},
})
if err != nil {
t.Fatalf("wildcard registration did not match mapping keys: %v", err)
}
for name, raw := range loaded.Config["collections"].(map[string]any) {
if got := len(raw.(map[string]any)["prompts"].([]any)); got != 2 {
t.Errorf("wildcard result for %q has %d records", name, got)
}
}
}
func TestFacadeSequenceIdentityRejectsInvalidDefinitionsBeforeReadingSource(t *testing.T) {
tests := []struct {
name string
identity configkit.SequenceIdentity
wantPath string
}{
{
name: "index path",
identity: configkit.SequenceIdentityAt(configkit.Keys("records").Index(0), "id"),
wantPath: "/records/0",
},
{
name: "empty fields",
identity: configkit.SequenceIdentityAt(configkit.Keys("records")),
wantPath: "/records",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
reader := strings.NewReader("records: []\n")
_, err := configkit.Load(configkit.Reader("input.yml", reader), configkit.Options[map[string]any]{
Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{test.identity}},
})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != configkit.CodeOptionsInvalid || loadErr.Diagnostics()[0].Path.String() != test.wantPath {
t.Fatalf("definition diagnostics = %#v", loadErr.Diagnostics())
}
if reader.Len() == 0 {
t.Fatal("source was read before registration validation")
}
})
}
}
func TestFacadeSequenceIdentityOverlapFailsBeforeSourceAccess(t *testing.T) {
selector := configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey()}
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(selector, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"), configkit.SequenceLiteralKey("team"),
}, "slug"),
}}
reader := strings.NewReader("collections: {}\n")
_, err := configkit.Load(configkit.Reader("input.yml", reader), configkit.Options[map[string]any]{Definition: definition})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != configkit.CodeOptionsInvalid {
t.Fatalf("overlap diagnostics = %#v", loadErr.Diagnostics())
}
if reader.Len() == 0 {
t.Fatal("source was read before overlapping registrations were rejected")
}
}
func TestFacadeSequenceIdentityConstraintsApplyAtInheritedDestinations(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("entries"),
configkit.SequenceLiteralKey("child"),
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
loaded, err := configkit.Load(configkit.Bytes("main", []byte(`jobs:
entries:
parent:
collections:
team:
prompts: [{id: inherited, text: base}]
child:
extends: parent
collections:
team:
prompts: [{id: added, text: child}]
`)), configkit.Options[map[string]any]{Definition: definition})
if err != nil {
t.Fatalf("load with dynamic inherited destination: %v", err)
}
child := loaded.Config["entries"].(map[string]any)["child"].(map[string]any)
prompts := child["collections"].(map[string]any)["team"].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Fatalf("inherited dynamic collection = %#v, want parent and child records", prompts)
}
}
func TestFacadeSequenceIdentityLoadsUseIndependentScopes(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(),
}, "id"),
}}
for _, source := range []string{"first", "second"} {
loaded, err := configkit.Load(configkit.Mapping(source, map[string]any{"collections": map[string]any{
source: []any{map[string]any{"id": "reused"}},
}}), configkit.Options[map[string]any]{Definition: definition})
if err != nil || len(loaded.Config["collections"].(map[string]any)[source].([]any)) != 1 {
t.Fatalf("independent load %q = %#v, %v", source, loaded, err)
}
}
}
func requireFacadeLoadError(t *testing.T, err error) *configkit.LoadError {
t.Helper()
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("error = %v, want *LoadError", err)
}
return loadErr
}

View File

@@ -21,7 +21,7 @@ func TestFacadeNoSourcePreservesAbsenceWithSequencePolicy(t *testing.T) {
definition.Defaults = func() map[string]any { return map[string]any{"fallback": true} } definition.Defaults = func() map[string]any { return map[string]any{"fallback": true} }
} }
if registered { if registered {
definition.SequenceCompositions = []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}} definition.SequenceIdentities = []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}
} }
ordinary, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{Definition: definition}) ordinary, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{Definition: definition})
if err != nil { if err != nil {
@@ -51,7 +51,7 @@ func TestFacadeNoSourcePreservesAbsenceWithSequencePolicy(t *testing.T) {
} }
} }
func TestFacadeItemHelperHistoryPreservesTraversalOrder(t *testing.T) { func TestFacadeItemHelperHistoryPreservesScheduledPlacement(t *testing.T) {
t.Parallel() t.Parallel()
filesystem := configkit.NewFilesystem("history", fstest.MapFS{ filesystem := configkit.NewFilesystem("history", fstest.MapFS{
"main.yml": {Data: []byte("records: [{id: authored}]\nconfigkit: {imports: [{path: one.yml, destination: [records], placement: sequence-item}, {path: two.yml, destination: [records], placement: sequence-item}]}\n")}, "main.yml": {Data: []byte("records: [{id: authored}]\nconfigkit: {imports: [{path: one.yml, destination: [records], placement: sequence-item}, {path: two.yml, destination: [records], placement: sequence-item}]}\n")},
@@ -63,24 +63,22 @@ func TestFacadeItemHelperHistoryPreservesTraversalOrder(t *testing.T) {
"child-helper.yml": {Data: []byte("text: nested helper\n")}, "child-helper.yml": {Data: []byte("text: nested helper\n")},
}) })
result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[map[string]any]{ result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[map[string]any]{
Imports: true, Definition: configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}}, Imports: true, Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
sourceOrder := make(map[string]int)
sourceFiles := make(map[string]string) sourceFiles := make(map[string]string)
for index, source := range result.Sources() { for _, source := range result.Sources() {
sourceOrder[source.ID] = index
sourceFiles[source.ID] = source.Path sourceFiles[source.ID] = source.Path
} }
for _, test := range []struct { for _, test := range []struct {
path configkit.Path path configkit.Path
file, text string file, text string
}{ }{
{configkit.Keys("records").Index(1).Key("text"), "helper-one.yml", "first helper\n"}, {configkit.Keys("records").Index(0).Key("text"), "helper-one.yml", "first helper\n"},
{configkit.Keys("records").Index(2).Key("text"), "helper-two.yml", "second helper\n"}, {configkit.Keys("records").Index(1).Key("text"), "helper-two.yml", "second helper\n"},
{configkit.Keys("records").Index(1).Key("children").Index(0).Key("text"), "child-helper.yml", "nested helper\n"}, {configkit.Keys("records").Index(0).Key("children").Index(0).Key("text"), "child-helper.yml", "nested helper\n"},
} { } {
explanation, ok := result.Explain(test.path) explanation, ok := result.Explain(test.path)
if !ok || explanation.Effective.YAML != test.text || len(explanation.History) != 1 { if !ok || explanation.Effective.YAML != test.text || len(explanation.History) != 1 {
@@ -95,20 +93,6 @@ func TestFacadeItemHelperHistoryPreservesTraversalOrder(t *testing.T) {
t.Fatal("metadata and explanation disagree") t.Fatal("metadata and explanation disagree")
} }
} }
explanation, ok := result.Explain(configkit.Keys())
if !ok {
t.Fatal("root explanation missing")
}
previous := -1
for _, change := range explanation.History {
for _, location := range change.Origin.Locations {
current := sourceOrder[location.SourceID]
if current < previous {
t.Fatalf("source %s replayed out of traversal order", location.SourceID)
}
previous = current
}
}
} }
func TestFacadeRegisteredReplacementExplainsRemovedDescendants(t *testing.T) { func TestFacadeRegisteredReplacementExplainsRemovedDescendants(t *testing.T) {
@@ -123,8 +107,14 @@ func TestFacadeRegisteredReplacementExplainsRemovedDescendants(t *testing.T) {
{"removed field", []any{map[string]any{"id": "main"}}, configkit.Keys("records").Index(0).Key("text")}, {"removed field", []any{map[string]any{"id": "main"}}, configkit.Keys("records").Index(0).Key("text")},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Mapping("main", map[string]any{"records": test.upper})}, configkit.LayeredOptions[map[string]any]{ mainYAML := "configkit:\n sequences:\n - destination: [records]\n key: id\n mode: replace\n"
Definition: configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}}, if len(test.upper) == 0 {
mainYAML += "records: []\n"
} else {
mainYAML += "records: [{id: main}]\n"
}
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte(mainYAML))}, configkit.LayeredOptions[map[string]any]{
Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("lower", map[string]any{"records": []any{map[string]any{"id": "one", "text": "old"}, map[string]any{"id": "two"}}})}}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("lower", map[string]any{"records": []any{map[string]any{"id": "one", "text": "old"}, map[string]any{"id": "two"}}})}}}},
}) })
if err != nil { if err != nil {
@@ -153,14 +143,14 @@ func TestFacadeItemConflictDiagnosticsFollowEnclosingIndexes(t *testing.T) {
"bad.yml": {Data: []byte("id: bad\nconfigkit: {imports: [{path: helper.yml}]}\n")}, "bad.yml": {Data: []byte("id: bad\nconfigkit: {imports: [{path: helper.yml}]}\n")},
"helper.yml": {Data: []byte("id: duplicate-field\n")}, "helper.yml": {Data: []byte("id: duplicate-field\n")},
}) })
definition := configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}} definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}}
for _, test := range []struct { for _, test := range []struct {
name, file, path string name, file, path string
grouped, itemMain bool grouped, itemMain bool
}{ }{
{"preceding authored and imported items", "main.yml", "/records/2/id", false, false}, {"preceding authored and imported items", "main.yml", "/records/0/id", false, false},
{"nested item", "nested.yml", "/records/1/children/1/id", false, false}, {"nested item", "nested.yml", "/records/0/children/0/id", false, false},
{"preceding grouped input", "group.yml", "/records/2/id", true, false}, {"preceding grouped input", "group.yml", "/records/1/id", true, false},
{"item main source", "bad.yml", "/records/0/id", false, true}, {"item main source", "bad.yml", "/records/0/id", false, true},
{"grouped item source", "bad.yml", "/records/1/id", true, true}, {"grouped item source", "bad.yml", "/records/1/id", true, true},
} { } {
@@ -193,8 +183,16 @@ func TestFacadeItemConflictDiagnosticsFollowEnclosingIndexes(t *testing.T) {
for _, source := range loadError.Sources() { for _, source := range loadError.Sources() {
files[source.ID] = source.Path files[source.ID] = source.Path
} }
if diagnostic.Primary == nil || files[diagnostic.Primary.SourceID] != "bad.yml" || len(diagnostic.Related) != 1 || files[diagnostic.Related[0].SourceID] != "helper.yml" { locations := append([]configkit.Location(nil), diagnostic.Related...)
t.Fatal("conflict lost physical source locations") if diagnostic.Primary != nil {
locations = append(locations, *diagnostic.Primary)
}
gotFiles := make(map[string]bool, len(locations))
for _, location := range locations {
gotFiles[files[location.SourceID]] = location.Line == 1 && location.Column == 5
}
if len(locations) != 2 || !gotFiles["bad.yml"] || !gotFiles["helper.yml"] {
t.Fatalf("conflict lost physical source locations: diagnostic=%#v sources=%#v", diagnostic, files)
} }
}) })
} }
@@ -202,7 +200,7 @@ func TestFacadeItemConflictDiagnosticsFollowEnclosingIndexes(t *testing.T) {
func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) { func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) {
definition := configkit.Definition[map[string]any]{ definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("entries"), IdentityField: "id"}}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
} }
ordinary, err := configkit.Load(configkit.Mapping("ordinary", map[string]any{"id": "one"}), configkit.Options[map[string]any]{ ordinary, err := configkit.Load(configkit.Mapping("ordinary", map[string]any{"id": "one"}), configkit.Options[map[string]any]{
Definition: definition, Destination: configkit.Keys("entries"), PlacementMode: configkit.PlacementModeSequenceItem, Definition: definition, Destination: configkit.Keys("entries"), PlacementMode: configkit.PlacementModeSequenceItem,
@@ -248,8 +246,8 @@ func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("registered sequence precedence load: %v", err) t.Fatalf("registered sequence precedence load: %v", err)
} }
if got := precedence.Config["entries"].([]any); len(got) != 1 || got[0].(map[string]any)["id"] != "main" { if got := precedence.Config["entries"].([]any); len(got) != 2 || got[0].(map[string]any)["id"] != "lower" || got[1].(map[string]any)["id"] != "main" {
t.Fatalf("higher-priority list did not replace the lower sequence: %#v", precedence.Config) t.Fatalf("registered upsert did not retain the lower sequence: %#v", precedence.Config)
} }
for _, test := range []struct { for _, test := range []struct {
@@ -303,7 +301,7 @@ items:
result, err := configkit.Load(configkit.File(mainPath), configkit.Options[map[string]any]{ result, err := configkit.Load(configkit.File(mainPath), configkit.Options[map[string]any]{
Imports: true, Imports: true,
Definition: configkit.Definition[map[string]any]{ Definition: configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("items"), IdentityField: "id"}}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Resolve: func(_ *map[string]any, resolution *configkit.Resolution) error { Resolve: func(_ *map[string]any, resolution *configkit.Resolution) error {
for _, id := range []string{"source-2", "source-3"} { for _, id := range []string{"source-2", "source-3"} {
source, ok := resolution.Source(id) source, ok := resolution.Source(id)
@@ -326,7 +324,7 @@ items:
t.Fatalf("load item glob: %v", err) t.Fatalf("load item glob: %v", err)
} }
items := result.Config["items"].([]any) items := result.Config["items"].([]any)
wantIDs := []string{"authored", "imported-a", "imported-b"} wantIDs := []string{"imported-a", "imported-b", "authored"}
for index, item := range items { for index, item := range items {
if got := item.(map[string]any)["id"]; got != wantIDs[index] { if got := item.(map[string]any)["id"]; got != wantIDs[index] {
t.Errorf("item %d ID = %#v, want %q", index, got, wantIDs[index]) t.Errorf("item %d ID = %#v, want %q", index, got, wantIDs[index])
@@ -344,10 +342,10 @@ items:
} }
} }
metadata := result.Metadata() metadata := result.Metadata()
for index := 1; index < len(wantIDs); index++ { for index := 0; index < 2; index++ {
path := configkit.Keys("items").Index(index).Key("id") path := configkit.Keys("items").Index(index).Key("id")
info, ok := metadata.Lookup(path) info, ok := metadata.Lookup(path)
if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != []string{"source-2", "source-3"}[index-1] { if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != []string{"source-2", "source-3"}[index] {
t.Errorf("shifted item origin at %s = %#v, %v", path.String(), info, ok) t.Errorf("shifted item origin at %s = %#v, %v", path.String(), info, ok)
} }
} }
@@ -463,15 +461,17 @@ func TestFacadeNestedItemFallbackUsesNearestResolvedContainer(t *testing.T) {
} }
} }
func TestFacadeSequenceCompositionRegistrationPathIsDetached(t *testing.T) { func TestFacadeSequenceIdentityRegistrationIsDetached(t *testing.T) {
registrations := []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}} fields := []string{"id"}
path := configkit.Keys("records")
registration := configkit.SequenceIdentityAt(path, fields...)
fields[0] = "changed"
definition := configkit.Definition[map[string]any]{ definition := configkit.Definition[map[string]any]{
SequenceCompositions: registrations, SequenceIdentities: []configkit.SequenceIdentity{registration},
Version: &configkit.VersionRule{ Version: &configkit.VersionRule{
Path: configkit.Keys("version"), Path: configkit.Keys("version"),
Check: func(configkit.Version, *configkit.Validation) error { Check: func(configkit.Version, *configkit.Validation) error {
registrations[0].Path = configkit.Keys("other") path = configkit.Keys("other")
registrations[0].IdentityField = "changed"
return nil return nil
}, },
}, },
@@ -514,6 +514,31 @@ func TestFacadeSourceInfoPlacementCopiesAndZeroValues(t *testing.T) {
} }
} }
func TestLayeredSequenceUpsertHistoryRetainsLowerTierRecords(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
}
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte("entries: [{id: main}]\n"))}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: configkit.Bytes("lower", []byte("entries: [{id: lower}]\n")),
}}}},
})
if err != nil {
t.Fatal(err)
}
for index, id := range []string{"lower", "main"} {
path := configkit.Keys("entries").Index(index).Key("id")
explanation, ok := result.Explain(path)
if !ok || !explanation.Effective.Present || explanation.Effective.YAML != id+"\n" || len(explanation.History) == 0 {
t.Fatalf("tier history at %s = %#v, %v", path.String(), explanation, ok)
}
if explanation.History[len(explanation.History)-1].After.YAML != id+"\n" {
t.Fatalf("tier history at %s does not retain final record: %#v", path.String(), explanation.History)
}
}
}
func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *testing.T) { func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *testing.T) {
type record struct { type record struct {
ID string `yaml:"id"` ID string `yaml:"id"`
@@ -531,7 +556,12 @@ func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *test
directory := t.TempDir() directory := t.TempDir()
mainPath := filepath.Join(directory, "main.yml") mainPath := filepath.Join(directory, "main.yml")
underlayPath := filepath.Join(directory, "underlay.yml") underlayPath := filepath.Join(directory, "underlay.yml")
if err := os.WriteFile(mainPath, []byte(`entries: if err := os.WriteFile(mainPath, []byte(`configkit:
sequences:
- destination: [entries]
key: id
mode: replace
entries:
- id: current - id: current
count: 0 count: 0
enabled: false enabled: false
@@ -567,7 +597,7 @@ func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *test
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.File(mainPath)}, configkit.LayeredOptions[configuration]{ result, err := configkit.LoadLayered(configkit.Input{Source: configkit.File(mainPath)}, configkit.LayeredOptions[configuration]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.File(underlayPath), Imports: true}}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.File(underlayPath), Imports: true}}}},
Definition: configkit.Definition[configuration]{ Definition: configkit.Definition[configuration]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("entries"), IdentityField: "id"}}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
Secret: func(path configkit.Path) bool { Secret: func(path configkit.Path) bool {
return path.String() == "/entries/0/token" || path.String() == "/entries/1/token" return path.String() == "/entries/0/token" || path.String() == "/entries/1/token"
}, },

View File

@@ -0,0 +1,197 @@
package configkit_test
import (
"fmt"
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
type movedVersionItem struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
}
type movedVersionCatalog struct {
Items []movedVersionItem `yaml:"items"`
}
func TestMainVersionDecodingFollowsSequenceMovement(t *testing.T) {
t.Parallel()
for _, mounted := range []bool{false, true} {
for _, matching := range []bool{false, true} {
for _, version := range []string{"4", "'4'"} {
t.Run(fmt.Sprintf("mounted=%t/matched=%t/%s", mounted, matching, version), func(t *testing.T) {
body := "items: [{id: main, version: " + version + "}]\n"
path := configkit.Keys("items").Index(0).Key("version")
main := configkit.Input{}
if mounted {
body, path = "id: main\nversion: "+version+"\n", configkit.Keys("version")
main.Destination, main.PlacementMode = configkit.Keys("items"), configkit.PlacementModeSequenceItem
}
main.Source = configkit.Bytes("main", []byte(body))
keep := "{id: keep}"
if mounted {
keep = "{id: keep, version: ordinary}"
}
lower := "items: [" + keep + "]\n"
if matching {
lower = "items: [" + keep + ", {id: main}]\n"
}
calls := 0
loaded, err := configkit.LoadLayered(main, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower", []byte(lower))}}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: path, Check: func(version configkit.Version, _ *configkit.Validation) error {
calls++
if !version.Present || version.Text != "4" {
t.Errorf("checked version = %#v", version)
}
return nil
}},
},
})
if err != nil {
t.Fatal(err)
}
if calls != 1 || len(loaded.Config.Items) != 2 || loaded.Config.Items[1].ID != "main" || loaded.Config.Items[1].Version != "4" {
t.Fatalf("moved main = %#v, checker calls = %d", loaded.Config, calls)
}
if mounted && loaded.Config.Items[0].Version != "ordinary" {
t.Fatal("unrelated underlay payload was treated as the main declaration")
}
for _, source := range loaded.Sources() {
if source.Layer == configkit.LayerBase && source.VersionCheck.Path.String() != path.String() {
t.Fatal("effective movement changed source-local version evidence")
}
}
})
}
}
}
}
func TestOverlayVersionProtectionUsesAssignedRecordPositions(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name, payload string
code configkit.Code
}{
{name: "change main header", payload: "[{id: main, version: changed}]", code: configkit.CodeOverlayVersion},
{name: "remove main header", payload: "[{id: main}]", code: configkit.CodeOverlayVersion},
{name: "other record version", payload: "[{id: keep, version: ordinary}]"},
{name: "new record version", payload: "[{id: new, version: ordinary}]"},
{name: "only main header gets integer conversion", payload: "[{id: new, version: 5}]", code: configkit.CodeDecodeType},
} {
t.Run(test.name, func(t *testing.T) {
files := configkit.NewFilesystem("catalog", fstest.MapFS{
"overlays/test.yml": {Data: []byte("items: " + test.payload + "\n")},
})
calls := 0
loaded, err := configkit.LoadLayered(configkit.Input{
Source: configkit.Bytes("main", []byte("id: main\nversion: 4\n")), Destination: configkit.Keys("items"), PlacementMode: configkit.PlacementModeSequenceItem,
}, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Overlay: "test", OverlayFilesystem: &configkit.FilesystemContext{FS: &files, BaseDir: ".", Root: "."},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower", []byte("items: [{id: keep}]\n"))}}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
if calls != 1 {
t.Fatalf("checker calls = %d", calls)
}
if test.code != "" {
loadErr := requireFacadeLoadError(t, err)
if loaded != nil || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != test.code {
t.Fatalf("version diagnostics = %#v", loadErr.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
if loaded.Config.Items[1].Version != "4" {
t.Fatal("unrelated overlay changed main header")
}
})
}
}
func TestUnderlayVersionProtectionUsesMainRecordPosition(t *testing.T) {
t.Parallel()
for _, imported := range []bool{false, true} {
t.Run(fmt.Sprintf("imported=%t", imported), func(t *testing.T) {
payload := "items: [{id: keep}, {id: main, version: forbidden}]\n"
lower := configkit.Input{Source: configkit.Bytes("lower", []byte(payload))}
want := configkit.CodeUnderlayVersion
if imported {
files := configkit.NewFilesystem("lower", fstest.MapFS{
"main.yml": {Data: []byte("configkit: {imports: [{path: fragment.yml}]}\n")},
"fragment.yml": {Data: []byte(payload)},
})
lower.Source, lower.Imports = files.File("main.yml"), true
want = configkit.CodeImportVersion
}
calls := 0
result, err := configkit.LoadLayered(configkit.Input{
Source: configkit.Bytes("main", []byte("id: main\nversion: 4\n")),
Destination: configkit.Keys("items"), PlacementMode: configkit.PlacementModeSequenceItem,
}, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{lower}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
failure := requireFacadeLoadError(t, err)
if result != nil || calls != 1 || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != want || failure.Diagnostics()[0].Path.String() != "/items/1/version" {
t.Fatalf("result=%#v calls=%d diagnostics=%#v", result, calls, failure.Diagnostics())
}
})
}
}
func TestMainVersionFollowsInheritedSequenceMovement(t *testing.T) {
t.Parallel()
for _, parentID := range []string{"keep", "main"} {
t.Run(parentID, func(t *testing.T) {
body := "base: {items: [{id: " + parentID + ", version: ordinary}]}\nchild:\n extends: base\n items: [{id: main, version: 4}]\n"
type config struct {
Base, Child movedVersionCatalog
}
calls := 0
result, err := configkit.Load(configkit.Bytes("main", []byte(body)), configkit.Options[config]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Definition: configkit.Definition[config]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("base", "items"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("child", "items"), "id"),
},
Version: &configkit.VersionRule{Path: configkit.Keys("child", "items").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
if calls != 1 {
t.Fatalf("checker calls = %d", calls)
}
if parentID == "main" {
failure := requireFacadeLoadError(t, err)
if result != nil || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != configkit.CodeInheritanceVersion {
t.Fatalf("diagnostics = %#v", failure.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
if len(result.Config.Child.Items) != 2 || result.Config.Child.Items[1].Version != "4" || result.Config.Child.Items[0].Version != "ordinary" {
t.Fatalf("inherited items = %#v", result.Config.Child.Items)
}
})
}
}

View File

@@ -13,15 +13,19 @@ import (
configkit "gitea.maximumdirect.net/eric/ConfigKit" configkit "gitea.maximumdirect.net/eric/ConfigKit"
) )
func TestFacadeSourceVersionEvidenceFollowsAuthoredSequenceRanges(t *testing.T) { func TestFacadeSourceVersionEvidenceFollowsAssembledSequenceRanges(t *testing.T) {
t.Parallel() t.Parallel()
for _, grouped := range []bool{false, true} { for _, grouped := range []bool{false, true} {
for _, sensitivePath := range []string{"", "/records/1/version", "/records/1"} { index := 0
if grouped {
index = 1
}
for _, sensitivePath := range []string{"", fmt.Sprintf("/records/%d/version", index), fmt.Sprintf("/records/%d", index)} {
t.Run(fmt.Sprintf("grouped=%t/secret=%s", grouped, sensitivePath), func(t *testing.T) { t.Run(fmt.Sprintf("grouped=%t/secret=%s", grouped, sensitivePath), func(t *testing.T) {
definition := configkit.Definition[map[string]any]{ definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("records").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { return nil }}, Version: &configkit.VersionRule{Path: configkit.Keys("records").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { return nil }},
Secret: func(path configkit.Path) bool { return sensitivePath != "" && path.String() == sensitivePath }, Secret: func(path configkit.Path) bool { return sensitivePath != "" && path.String() == sensitivePath },
} }
first := "records: [{id: first, version: first-version}]\n" first := "records: [{id: first, version: first-version}]\n"
child := "records: [{id: child, version: PRIVATE-VERSION-CANARY}]\n" child := "records: [{id: child, version: PRIVATE-VERSION-CANARY}]\n"
@@ -46,6 +50,16 @@ func TestFacadeSourceVersionEvidenceFollowsAuthoredSequenceRanges(t *testing.T)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !grouped {
records, ok := result.Config["records"].([]any)
if !ok || len(records) != 2 {
t.Fatalf("scheduled records = %#v", result.Config["records"])
}
firstRecord, ok := records[0].(map[string]any)
if !ok || firstRecord["id"] != "child" {
t.Fatalf("scheduled records = %#v", result.Config["records"])
}
}
sources := result.Sources() sources := result.Sources()
check := sources[len(sources)-1].VersionCheck check := sources[len(sources)-1].VersionCheck
if check == nil || check.Path.String() != "/records/0/version" || check.Location == nil || check.Location.SourceID != sources[len(sources)-1].ID { if check == nil || check.Path.String() != "/records/0/version" || check.Location == nil || check.Location.SourceID != sources[len(sources)-1].ID {
@@ -72,8 +86,8 @@ func TestFacadeSourceVersionOwnershipAcceptsContextualIndexes(t *testing.T) {
for _, grouped := range []bool{false, true} { for _, grouped := range []bool{false, true} {
for _, path := range []configkit.Path{configkit.Keys("records").Index(0).Key("version"), parsed} { for _, path := range []configkit.Path{configkit.Keys("records").Index(0).Key("version"), parsed} {
definition := configkit.Definition[map[string]any]{ definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")},
Version: &configkit.VersionRule{Path: path, Check: func(configkit.Version, *configkit.Validation) error { return nil }}, Version: &configkit.VersionRule{Path: path, Check: func(configkit.Version, *configkit.Validation) error { return nil }},
} }
first := "records: [{id: first, version: first}]\n" first := "records: [{id: first, version: first}]\n"
child := "records: [{id: child, version: child}]\n" child := "records: [{id: child, version: child}]\n"
@@ -90,11 +104,7 @@ func TestFacadeSourceVersionOwnershipAcceptsContextualIndexes(t *testing.T) {
if err != nil || result == nil { if err != nil || result == nil {
t.Fatalf("grouped=%t path=%#v: %v", grouped, path, err) t.Fatalf("grouped=%t path=%#v: %v", grouped, path, err)
} }
want := 2 if len(result.Config["records"].([]any)) != 2 {
if grouped {
want = 1
}
if len(result.Config["records"].([]any)) != want {
t.Fatal("version path changed composition") t.Fatal("version path changed composition")
} }
} }
@@ -626,7 +636,7 @@ func TestFacadeSourceVersionEvidenceOnFailedLoadsAndSyntheticSources(t *testing.
Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { checked++; return nil }, Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { checked++; return nil },
}} }}
path := filepath.Join(t.TempDir(), "main.yml") path := filepath.Join(t.TempDir(), "main.yml")
contents := "version: " + canary + "\nconfigkit:\n imports:\n - path: missing.yml\n" contents := "version: " + canary + "\nconfigkit:\n imports:\n - path: missing.yml\n required: true\n"
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -0,0 +1,106 @@
package engine
import (
"errors"
"fmt"
"testing"
"testing/fstest"
)
func TestNestedCollectionInitializationIsIndependentOfAncestorPresence(t *testing.T) {
t.Parallel()
for _, ancestor := range []string{"{}", "group: {}", "group: null"} {
for _, test := range []struct {
mode, key, value string
valid bool
}{
{mode: "append", value: "[{id: a}]"},
{mode: "upsert", value: "[{id: a}]"},
{mode: "append", value: "[]"},
{mode: "upsert", value: "[]"},
{mode: "append", key: ", key: id", value: "{}"},
{mode: "upsert", key: ", key: id", value: "null"},
{mode: "append", key: ", key: id", value: "[{id: a}]", valid: true},
{mode: "upsert", key: ", key: id", value: "[]", valid: true},
} {
t.Run(fmt.Sprintf("%s/%s/%s/%s", ancestor, test.mode, test.key, test.value), func(t *testing.T) {
main := "configkit:\n sequences: [{destination: [group, items], mode: " + test.mode + test.key + "}]\ngroup: {items: " + test.value + "}\n"
loaded, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(ancestor))}}}},
})
if test.valid {
if err != nil || loaded == nil {
t.Fatalf("valid initialization failed: %v", err)
}
return
}
var loadErr *LoadError
if !errors.As(err, &loadErr) || loaded != nil {
t.Fatalf("invalid collection operation succeeded: %#v, %v", loaded, err)
}
diagnostics := loadErr.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/group/items" {
t.Fatalf("operation diagnostics = %#v", diagnostics)
}
})
}
}
}
func TestAddedAncestorTransfersNestedRecordPolicies(t *testing.T) {
t.Parallel()
type record struct {
ID string `yaml:"id"`
Steps []map[string]any `yaml:"steps"`
}
type config struct {
Group struct {
Items []record `yaml:"items"`
} `yaml:"group"`
}
files := fstest.MapFS{
"main.yml": {Data: []byte(`configkit:
sequences: [{destination: [group, items], key: id}]
imports: [{path: base.yml, destination: [group, items], placement: sequence-item}]
group:
items: [{id: child, extends: {id: base}, steps: [{name: child}]}]
`)},
"base.yml": {Data: []byte("configkit:\n sequences: [{destination: [steps], key: name}]\nid: base\nsteps: [{name: base}]\n")},
}
loaded, err := testGroupedLoad(loadRequest[config]{
main: loadInput{source: NewFilesystem("catalog", files).File("main.yml"), imports: true},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("{}"))}}}},
})
if err != nil {
t.Fatal(err)
}
for _, item := range loaded.Config.Group.Items {
if item.ID == "child" {
if len(item.Steps) != 2 || item.Steps[0]["name"] != "base" || item.Steps[1]["name"] != "child" {
t.Fatalf("record-local identity was lost when adding its ancestor: %#v", item.Steps)
}
return
}
}
t.Fatal("missing child record")
}
func TestInheritanceChecksCollectionOperationUnderAddedAncestor(t *testing.T) {
t.Parallel()
_, err := testLoad[map[string]any](Bytes("inheritance", []byte(`configkit:
sequences: [{destination: [child, group, items], mode: append}]
base: {}
child:
extends: base
group: {items: [{id: a}]}
`)))
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("missing nested operation error: %v", err)
}
diagnostics := loadErr.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/child/group/items" {
t.Fatalf("nested inheritance operation diagnostics = %#v", diagnostics)
}
}

View File

@@ -0,0 +1,98 @@
package engine
import (
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestOverlayBroadCollectionPoliciesAndRetainedDeclarations(t *testing.T) {
var baseBody, overlayBody strings.Builder
basePolicies := collectionPolicyScope{}
var declarations []sequenceDeclarationSpec
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
fmt.Fprintf(&baseBody, "%s: {items: [{id: base}], retained: [{id: retained}], scalar: pending}\n", name)
fmt.Fprintf(&overlayBody, "%s: {items: [{id: overlay}], added: true}\n", name)
basePolicies.set(Keys(name, "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
})
for _, key := range []string{"retained", "missing", "scalar"} {
declarations = append(declarations, sequenceDeclarationSpec{
destination: Keys(name, key), identityFields: []string{"id"}, identityPresent: true,
mode: sequenceModeAppend, modePresent: true,
})
}
}
base, diagnostics := composeDocuments(parseCompositionDocument(t, "base", baseBody.String()))
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
base.collectionPolicies = basePolicies
assembly := newAssemblyAccumulator(base)
overlay := parseCompositionDocument(t, "overlay", overlayBody.String())
diagnostics = assembly.applyOverlayWithCollectionPolicy(overlay, nil, collectionPolicyScope{}, nil, LayerOverlay, sequencePolicy{}, declarations)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
lookup := buildDocumentLookup(assembly.current.root)
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
items, found := documentAtPathWithLookup(assembly.current.root, Keys(name, "items"), lookup)
if !found || items.Kind != document.Sequence || len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "overlay" {
t.Fatalf("collection %s did not append in order", name)
}
for _, key := range []string{"retained", "missing", "scalar"} {
policy, found := assembly.current.collectionPolicies.at(Keys(name, key))
if !found || !policy.identityPresent || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
t.Fatalf("retained/pending declaration lost at %s/%s", name, key)
}
}
if _, found := documentAtPathWithLookup(assembly.current.root, Keys(name, "missing"), lookup); found {
t.Fatal("pending declaration created a value")
}
if origin := assembly.current.origins[Keys(name, "items").Index(1).Key("id").String()]; origin.Layer != LayerOverlay {
t.Fatal("appended item lost overlay attribution")
}
}
}
func TestInheritanceBroadCollectionsInOneDefinition(t *testing.T) {
var body strings.Builder
policies := collectionPolicyScope{}
for _, definition := range []string{"base", "child"} {
fmt.Fprintf(&body, "%s:\n", definition)
if definition == "child" {
body.WriteString(" extends: base\n")
}
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
fmt.Fprintf(&body, " %s: {items: [{id: %s}]}\n", name, definition)
policy := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
if definition == "child" {
policy.explicitMode, policy.explicitModePresent = sequenceModeAppend, true
}
policies.set(Keys(definition, name, "items"), policy)
policies.set(Keys(definition, name, "pending"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
}
}
root := parseCompositionDocument(t, "wide-collections", body.String())
result, diagnostics := expandInheritanceWithPolicies(root, nil, policies, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
lookup := buildDocumentLookup(result.root)
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
items, found := documentAtPathWithLookup(result.root, Keys("child", name, "items"), lookup)
if !found || len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "child" {
t.Fatalf("inherited collection %s did not append in order", name)
}
if policy, found := result.collectionPolicies.at(Keys("child", name, "pending")); !found || !policy.identityPresent {
t.Fatalf("pending inherited policy lost at %s", name)
}
}
}

View File

@@ -0,0 +1,209 @@
package engine
type collectionPolicy struct {
identityFields []string
identityPresent bool
identityLocation *Location
explicitMode sequenceMode
explicitModePresent bool
modeLocation *Location
recordOrder []*collectionRecordOwner
recordPolicies map[*collectionRecordOwner]collectionPolicyScope
}
type collectionPolicyResolution struct {
policy collectionPolicy
operation sequenceMode
declarations []sequenceDeclarationSpec
declarationsPresent bool
identityDeclarationPresent bool
modeDeclarationPresent bool
applicationIdentityFields []string
applicationIdentityPresent bool
effectiveIdentityFields []string
}
// resolveCollectionPolicy reconciles declarations for one destination before
// peers are composed. The caller supplies only lower policy that is applicable
// to this destination; record survival and ownership are resolved elsewhere.
func resolveCollectionPolicy(
destination Path,
declarations []sequenceDeclarationSpec,
applicationIdentityFields []string,
lower *collectionPolicy,
) (collectionPolicyResolution, []Diagnostic) {
normalizedDeclarations := cloneSequenceDeclarations(declarations)
for index := range normalizedDeclarations {
declaration := &normalizedDeclarations[index]
if !declaration.identityPresent {
continue
}
fields, err := normalizeSequenceIdentityFields(declaration.identityFields)
if err != nil {
return collectionPolicyResolution{}, []Diagnostic{{
Code: CodeSequenceInvalid, Severity: SeverityError, Path: destination.clone(),
Message: "collection identity declaration requires at least one field",
Primary: collectionDeclarationLocation(declaration.identityLocation, declaration.location),
}}
}
declaration.identityFields = fields
}
normalizedApplicationIdentityFields := cloneStrings(applicationIdentityFields)
if len(applicationIdentityFields) != 0 {
var err error
normalizedApplicationIdentityFields, err = normalizeSequenceIdentityFields(applicationIdentityFields)
if err != nil {
return collectionPolicyResolution{}, []Diagnostic{{
Code: CodeOptionsInvalid, Severity: SeverityError, Path: destination.clone(),
Message: "application collection identity fields are invalid",
}}
}
}
result := collectionPolicyResolution{
declarations: normalizedDeclarations,
declarationsPresent: len(normalizedDeclarations) != 0,
applicationIdentityFields: normalizedApplicationIdentityFields,
applicationIdentityPresent: len(normalizedApplicationIdentityFields) != 0,
}
var diagnostics []Diagnostic
var declaredIdentityFields []string
var declaredIdentityLocation *Location
var declaredMode sequenceMode
var declaredModeLocation *Location
for _, declaration := range normalizedDeclarations {
if declaration.identityPresent {
result.identityDeclarationPresent = true
if declaredIdentityFields == nil {
declaredIdentityFields = cloneStrings(declaration.identityFields)
declaredIdentityLocation = collectionDeclarationLocation(declaration.identityLocation, declaration.location)
} else if !equalSequenceIdentityFields(declaredIdentityFields, declaration.identityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(declaration.identityLocation, declaration.location),
"collection identity declarations conflict",
))
}
}
if declaration.modePresent {
result.modeDeclarationPresent = true
if declaredMode == 0 {
declaredMode = declaration.mode
declaredModeLocation = collectionDeclarationLocation(declaration.modeLocation, declaration.location)
} else if declaredMode != declaration.mode {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredModeLocation,
collectionDeclarationLocation(declaration.modeLocation, declaration.location),
"collection operation declarations conflict",
))
}
}
}
if len(diagnostics) != 0 {
return collectionPolicyResolution{}, canonicalDiagnostics(diagnostics)
}
if lower != nil && lower.identityPresent && declaredIdentityFields != nil && !equalSequenceIdentityFields(lower.identityFields, declaredIdentityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(lower.identityLocation, Location{}),
"collection identity declaration conflicts with retained identity",
))
}
if result.applicationIdentityPresent {
if declaredIdentityFields != nil && !equalSequenceIdentityFields(normalizedApplicationIdentityFields, declaredIdentityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation, nil,
"collection identity declaration conflicts with application constraint",
))
}
if lower != nil && lower.identityPresent && !equalSequenceIdentityFields(normalizedApplicationIdentityFields, lower.identityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(lower.identityLocation, Location{}),
"retained collection identity conflicts with application constraint",
))
}
}
if len(diagnostics) != 0 {
return collectionPolicyResolution{}, canonicalDiagnostics(diagnostics)
}
if declaredIdentityFields != nil {
result.policy.identityFields = declaredIdentityFields
result.policy.identityPresent = true
result.policy.identityLocation = cloneLocation(declaredIdentityLocation)
} else if lower != nil && lower.identityPresent {
result.policy.identityFields = cloneStrings(lower.identityFields)
result.policy.identityPresent = true
result.policy.identityLocation = cloneLocation(lower.identityLocation)
}
if result.applicationIdentityPresent {
result.effectiveIdentityFields = cloneStrings(normalizedApplicationIdentityFields)
} else if result.policy.identityPresent {
result.effectiveIdentityFields = cloneStrings(result.policy.identityFields)
}
if result.modeDeclarationPresent {
result.policy.explicitMode = declaredMode
result.policy.explicitModePresent = true
result.policy.modeLocation = cloneLocation(declaredModeLocation)
} else if lower != nil && lower.explicitModePresent {
result.policy.explicitMode = lower.explicitMode
result.policy.explicitModePresent = true
result.policy.modeLocation = cloneLocation(lower.modeLocation)
}
result.operation = collectionOperationMode(declaredMode, result.policy.explicitMode, len(result.effectiveIdentityFields) != 0)
return result, nil
}
func collectionOperationMode(current, retained sequenceMode, identified bool) sequenceMode {
if current != 0 {
return current
}
if retained != 0 {
return retained
}
if identified {
return sequenceModeUpsert
}
return sequenceModeReplace
}
func cloneSequenceDeclarations(declarations []sequenceDeclarationSpec) []sequenceDeclarationSpec {
if declarations == nil {
return nil
}
cloned := make([]sequenceDeclarationSpec, len(declarations))
for index, declaration := range declarations {
cloned[index] = declaration
cloned[index].path = declaration.path.clone()
cloned[index].destination = declaration.destination.clone()
cloned[index].identityFields = cloneStrings(declaration.identityFields)
cloned[index].identityLocation = cloneLocation(declaration.identityLocation)
cloned[index].modeLocation = cloneLocation(declaration.modeLocation)
}
return cloned
}
func collectionDeclarationLocation(specific *Location, fallback Location) *Location {
if specific != nil {
return cloneLocation(specific)
}
if fallback != (Location{}) {
return cloneLocation(&fallback)
}
return nil
}
func collectionPolicyConflictDiagnostic(path Path, primary, related *Location, message string) Diagnostic {
diagnostic := Diagnostic{
Code: CodeSequenceConflict, Severity: SeverityError,
Path: path.clone(), Message: message, Primary: cloneLocation(primary),
}
if related != nil {
diagnostic.Related = []Location{*cloneLocation(related)}
}
return diagnostic
}

View File

@@ -0,0 +1,258 @@
package engine
import (
"sort"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
type collectionPolicyScope struct {
collections map[string]collectionPolicyEntry
}
type collectionPolicyEntry struct {
destination Path
policy collectionPolicy
}
type collectionRecordPolicy struct {
owner *collectionRecordOwner
scope collectionPolicyScope
}
// The nonzero field ensures simultaneously live record owners have distinct
// addresses. The token is per-record ownership, not a source or identity key.
type collectionRecordOwner struct {
marker byte
}
type projectedCollectionPolicy struct {
path Path
policy collectionPolicy
}
func newCollectionRecordPolicy() collectionRecordPolicy {
return collectionRecordPolicy{owner: &collectionRecordOwner{marker: 1}}
}
func retainedCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
return collectionRecordPolicy{
owner: record.owner,
scope: cloneCollectionPolicyScope(record.scope),
}
}
func replacedCollectionRecordPolicy() collectionRecordPolicy {
return newCollectionRecordPolicy()
}
func (policy *collectionPolicy) setRecordPolicy(record collectionRecordPolicy) {
if record.owner == nil {
record.owner = newCollectionRecordPolicy().owner
}
if policy.recordPolicies == nil {
policy.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope)
}
if _, exists := policy.recordPolicies[record.owner]; !exists {
policy.recordOrder = append(policy.recordOrder, record.owner)
}
policy.recordPolicies[record.owner] = cloneCollectionPolicyScope(record.scope)
}
func (policy *collectionPolicy) setRecordPolicyAt(index int, record collectionRecordPolicy) {
if index < 0 {
return
}
if record.owner == nil {
record.owner = newCollectionRecordPolicy().owner
}
if policy.recordPolicies == nil {
policy.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope)
}
for len(policy.recordOrder) <= index {
owner := newCollectionRecordPolicy().owner
policy.recordOrder = append(policy.recordOrder, owner)
policy.recordPolicies[owner] = collectionPolicyScope{}
}
delete(policy.recordPolicies, policy.recordOrder[index])
policy.recordOrder[index] = record.owner
policy.recordPolicies[record.owner] = cloneCollectionPolicyScope(record.scope)
}
func (policy collectionPolicy) recordPolicyAt(index int) (collectionRecordPolicy, bool) {
if index < 0 || index >= len(policy.recordOrder) {
return collectionRecordPolicy{}, false
}
owner := policy.recordOrder[index]
scope, ok := policy.recordPolicies[owner]
if !ok {
return collectionRecordPolicy{}, false
}
return collectionRecordPolicy{owner: owner, scope: cloneCollectionPolicyScope(scope)}, true
}
func collectionPolicyForReplacement(lower *collectionPolicy, current collectionPolicy) collectionPolicy {
result := cloneCollectionPolicy(current)
if !current.explicitModePresent && lower != nil && lower.explicitModePresent {
result.explicitMode = lower.explicitMode
result.explicitModePresent = true
result.modeLocation = cloneLocation(lower.modeLocation)
}
return result
}
// collectionPolicyWithoutValue keeps identity and record ownership because no
// collection value was replaced. A key-only declaration can remain pending
// when no sequence is present, or validate the retained sequence immediately.
func collectionPolicyWithoutValue(
lower *collectionPolicy,
current collectionPolicy,
retainedValue *document.Node,
path Path,
) (collectionPolicy, []Diagnostic) {
result := collectionPolicy{}
if lower != nil {
result = cloneCollectionPolicy(*lower)
}
if lower != nil && lower.identityPresent && current.identityPresent && !equalSequenceIdentityFields(lower.identityFields, current.identityFields) {
return collectionPolicy{}, []Diagnostic{collectionPolicyConflictDiagnostic(
path, current.identityLocation, lower.identityLocation,
"collection identity declaration conflicts with retained identity",
)}
}
if current.identityPresent {
result.identityFields = cloneStrings(current.identityFields)
result.identityPresent = true
result.identityLocation = cloneLocation(current.identityLocation)
}
if current.explicitModePresent {
result.explicitMode = current.explicitMode
result.explicitModePresent = true
result.modeLocation = cloneLocation(current.modeLocation)
}
if current.identityPresent && retainedValue != nil && retainedValue.Kind == document.Sequence {
_, diagnostics := validateSequenceIdentity(retainedValue, path, result.identityFields, sequenceIdentityDiagnosticCodes{
invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict,
})
if len(diagnostics) != 0 {
return collectionPolicy{}, diagnostics
}
}
return result, nil
}
func (scope *collectionPolicyScope) set(destination Path, policy collectionPolicy) {
if scope.collections == nil {
scope.collections = make(map[string]collectionPolicyEntry)
}
scope.collections[destination.String()] = collectionPolicyEntry{
destination: destination.clone(), policy: cloneCollectionPolicy(policy),
}
}
func (scope collectionPolicyScope) at(destination Path) (collectionPolicy, bool) {
entry, ok := scope.collections[destination.String()]
if !ok {
return collectionPolicy{}, false
}
return cloneCollectionPolicy(entry.policy), true
}
// retireCollectionPoliciesAtOrBelow clears YAML identity and record-owned state
// below a replaced ancestor while retaining independently scoped explicit modes.
func (scope *collectionPolicyScope) retireCollectionPoliciesAtOrBelow(ancestor Path) {
if scope == nil {
return
}
for key, entry := range scope.collections {
if !pathHasPrefix(entry.destination, ancestor) {
continue
}
clearCollectionIdentity(&entry.policy)
scope.collections[key] = entry
}
}
func projectCollectionPolicyScope(scope collectionPolicyScope, prefix Path) []projectedCollectionPolicy {
keys := make([]string, 0, len(scope.collections))
for key := range scope.collections {
keys = append(keys, key)
}
sort.Strings(keys)
var projected []projectedCollectionPolicy
for _, key := range keys {
entry := scope.collections[key]
concrete := appendDestination(prefix, entry.destination)
projected = append(projected, projectedCollectionPolicy{path: concrete, policy: cloneCollectionPolicy(entry.policy)})
for index, owner := range entry.policy.recordOrder {
if recordScope, ok := entry.policy.recordPolicies[owner]; ok {
projected = append(projected, projectCollectionPolicyScope(recordScope, concrete.Index(index))...)
}
}
}
return projected
}
func cloneCollectionPolicyScope(scope collectionPolicyScope) collectionPolicyScope {
if scope.collections == nil {
return collectionPolicyScope{}
}
cloned := collectionPolicyScope{collections: make(map[string]collectionPolicyEntry, len(scope.collections))}
for key, entry := range scope.collections {
cloned.collections[key] = collectionPolicyEntry{
destination: entry.destination.clone(), policy: cloneCollectionPolicy(entry.policy),
}
}
return cloned
}
func cloneCollectionPolicyScopeForInheritance(scope collectionPolicyScope) collectionPolicyScope {
if scope.collections == nil {
return collectionPolicyScope{}
}
cloned := collectionPolicyScope{collections: make(map[string]collectionPolicyEntry, len(scope.collections))}
for key, entry := range scope.collections {
cloned.collections[key] = collectionPolicyEntry{
destination: entry.destination.clone(), policy: cloneCollectionPolicyForInheritance(entry.policy),
}
}
return cloned
}
func cloneCollectionPolicy(policy collectionPolicy) collectionPolicy {
cloned := policy
cloned.identityFields = cloneStrings(policy.identityFields)
cloned.identityLocation = cloneLocation(policy.identityLocation)
cloned.modeLocation = cloneLocation(policy.modeLocation)
if policy.recordOrder != nil {
cloned.recordOrder = append([]*collectionRecordOwner(nil), policy.recordOrder...)
}
if policy.recordPolicies != nil {
cloned.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope, len(policy.recordPolicies))
for owner, scope := range policy.recordPolicies {
cloned.recordPolicies[owner] = cloneCollectionPolicyScope(scope)
}
}
return cloned
}
func cloneCollectionPolicyForInheritance(policy collectionPolicy) collectionPolicy {
cloned := policy
cloned.identityFields = cloneStrings(policy.identityFields)
cloned.identityLocation = cloneLocation(policy.identityLocation)
cloned.modeLocation = cloneLocation(policy.modeLocation)
if policy.recordOrder != nil {
cloned.recordOrder = make([]*collectionRecordOwner, 0, len(policy.recordOrder))
}
if policy.recordPolicies != nil {
cloned.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope, len(policy.recordPolicies))
for _, sourceOwner := range policy.recordOrder {
owner := newCollectionRecordPolicy().owner
cloned.recordOrder = append(cloned.recordOrder, owner)
if sourceScope, ok := policy.recordPolicies[sourceOwner]; ok {
cloned.recordPolicies[owner] = cloneCollectionPolicyScopeForInheritance(sourceScope)
}
}
}
return cloned
}

View File

@@ -0,0 +1,292 @@
package engine
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestCollectionPolicyWithoutValueRetainsIdentityAndMode(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
current := collectionPolicy{explicitMode: sequenceModeReplace, explicitModePresent: true}
retained, diagnostics := collectionPolicyWithoutValue(&lower, current, nil, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent || !reflect.DeepEqual(retained.identityFields, []string{"id"}) {
t.Fatalf("no-value replace reset identity: %#v, %#v", retained, diagnostics)
}
if retained.explicitMode != sequenceModeReplace || !retained.explicitModePresent || len(retained.recordOrder) != 1 {
t.Fatalf("no-value policy update lost mode or record ownership: %#v", retained)
}
retainedRecord, _ := retained.recordPolicyAt(0)
lowerRecord, _ := lower.recordPolicyAt(0)
if retainedRecord.owner != lowerRecord.owner {
t.Fatal("no-value policy update changed record ownership")
}
}
func TestCollectionPolicyWithoutValueLeavesAbsentAndNonSequenceDestinationsUnconstrained(t *testing.T) {
current := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
for _, value := range []*document.Node{nil, parsePolicyNode(t, "value: scalar\n").Mapping[0].Value} {
retained, diagnostics := collectionPolicyWithoutValue(nil, current, value, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent || !reflect.DeepEqual(retained.identityFields, []string{"id"}) {
t.Fatalf("pending identity at absent/non-sequence destination = %#v, %#v", retained, diagnostics)
}
}
}
func TestCollectionPolicyWithoutValueValidatesRetainedSequenceImmediately(t *testing.T) {
current := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
valid := parsePolicyNode(t, "items: [{id: one}, {id: two}]\n").Mapping[0].Value
retained, diagnostics := collectionPolicyWithoutValue(nil, current, valid, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent {
t.Fatalf("valid retained records were rejected: %#v, %#v", retained, diagnostics)
}
invalid := parsePolicyNode(t, "items: [{name: missing-id}]\n").Mapping[0].Value
retained, diagnostics = collectionPolicyWithoutValue(nil, current, invalid, Keys("prompts"))
if !reflect.DeepEqual(retained, collectionPolicy{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/prompts/0/id" {
t.Fatalf("invalid retained records = %#v, %#v", retained, diagnostics)
}
}
func TestCollectionPolicyWithoutValueRejectsIdentityChange(t *testing.T) {
lower := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
current := collectionPolicy{identityFields: []string{"name"}, identityPresent: true}
retained, diagnostics := collectionPolicyWithoutValue(&lower, current, nil, Keys("prompts"))
if !reflect.DeepEqual(retained, collectionPolicy{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("no-value identity change = %#v, %#v", retained, diagnostics)
}
}
func TestCollectionPolicyLifetimePersistenceAndReplacementTable(t *testing.T) {
tests := []struct {
name string
current collectionPolicy
replaceValue bool
wantIdentity []string
wantIdentitySet bool
wantMode sequenceMode
wantRecords int
}{
{
name: "mode-only replace declaration does not reset retained identity",
current: collectionPolicy{
explicitMode: sequenceModeReplace, explicitModePresent: true,
},
wantIdentity: []string{"id"}, wantIdentitySet: true,
wantMode: sequenceModeReplace, wantRecords: 1,
},
{
name: "whole-sequence replacement resets lower identity",
replaceValue: true,
wantIdentitySet: false,
wantMode: sequenceModeAppend,
wantRecords: 0,
},
{
name: "replacement can establish a new identity",
current: collectionPolicy{identityFields: []string{"name"}, identityPresent: true},
replaceValue: true,
wantIdentity: []string{"name"},
wantIdentitySet: true,
wantMode: sequenceModeAppend,
wantRecords: 0,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
var got collectionPolicy
if test.replaceValue {
got = collectionPolicyForReplacement(&lower, test.current)
} else {
var diagnostics []Diagnostic
got, diagnostics = collectionPolicyWithoutValue(&lower, test.current, nil, Keys("prompts"))
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
}
if got.identityPresent != test.wantIdentitySet || !reflect.DeepEqual(got.identityFields, test.wantIdentity) || got.explicitMode != test.wantMode || len(got.recordOrder) != test.wantRecords {
t.Fatalf("transition result = %#v", got)
}
})
}
}
func TestCollectionPolicyValueReplacementResetsIdentityButRetainsExplicitMode(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
current := collectionPolicy{identityFields: []string{"name"}, identityPresent: true}
replaced := collectionPolicyForReplacement(&lower, current)
if !replaced.identityPresent || !reflect.DeepEqual(replaced.identityFields, []string{"name"}) {
t.Fatalf("replacement identity was not reset: %#v", replaced)
}
if replaced.explicitMode != sequenceModeAppend || !replaced.explicitModePresent || len(replaced.recordOrder) != 0 {
t.Fatalf("replacement failed to retain explicit mode or retire lower record policy: %#v", replaced)
}
cleared := collectionPolicyForReplacement(&lower, collectionPolicy{})
if cleared.identityPresent || len(cleared.identityFields) != 0 || cleared.explicitMode != sequenceModeAppend || !cleared.explicitModePresent {
t.Fatalf("replacement without a new key did not clear YAML identity while retaining mode: %#v", cleared)
}
}
func TestCollectionRecordRetentionAndReplacementUseOpaqueOwners(t *testing.T) {
oldRecord := newCollectionRecordPolicy()
oldRecord.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
retained := retainedCollectionRecordPolicy(oldRecord)
if retained.owner != oldRecord.owner {
t.Fatal("retained record did not keep its owner")
}
if got, ok := retained.scope.at(Keys("steps")); !ok || !got.identityPresent {
t.Fatalf("retained record lost its local policy: %#v, %t", got, ok)
}
if retained.scope.collections["/steps"].policy.identityFields[0] = "changed"; oldRecord.scope.collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("retained record policy aliases its source")
}
// A whole-record replacement at the same list position, even with equal
// authored identity/source metadata, receives new ownership and no old rules.
replacement := replacedCollectionRecordPolicy()
if replacement.owner == oldRecord.owner || len(replacement.scope.collections) != 0 {
t.Fatal("replacement record inherited discarded owner policy")
}
}
func TestCollectionPolicyAncestorRetirementPreservesIndependentMode(t *testing.T) {
var scope collectionPolicyScope
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
scope.set(Keys("catalog", "prompts"), collectionPolicyWithRecords(collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}, owner))
scope.retireCollectionPoliciesAtOrBelow(Keys("catalog"))
policy, ok := scope.at(Keys("catalog", "prompts"))
if !ok || policy.identityPresent || len(policy.recordOrder) != 0 || policy.explicitMode != sequenceModeAppend || !policy.explicitModePresent {
t.Fatalf("ancestor retirement = %#v, %t", policy, ok)
}
reintroduced, diagnostics := collectionPolicyWithoutValue(&policy, collectionPolicy{}, nil, Keys("catalog", "prompts"))
if len(diagnostics) != 0 || reintroduced.identityPresent || reintroduced.explicitMode != sequenceModeAppend {
t.Fatalf("retired identity was resurrected while mode persisted: %#v, %#v", reintroduced, diagnostics)
}
}
func TestEqualOuterIdentitiesHaveDistinctRecordOwners(t *testing.T) {
retained := newCollectionRecordPolicy()
replacement := replacedCollectionRecordPolicy()
if retained.owner == replacement.owner {
t.Fatal("replacement with the same outer identity reused the old owner")
}
if retained.owner == nil || replacement.owner == nil {
t.Fatal("record owner token is missing")
}
}
func TestCollectionPolicyScopesKeepRootAndItemRulesIndependent(t *testing.T) {
var root collectionPolicyScope
root.set(Keys("steps"), collectionPolicy{identityFields: []string{"root-id"}, identityPresent: true})
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"item-id"}, identityPresent: true})
root.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, owner))
root.retireCollectionPoliciesAtOrBelow(Keys("steps"))
rootPolicy, ok := root.at(Keys("steps"))
if !ok || rootPolicy.identityPresent {
t.Fatalf("root-local policy was not retired: %#v", rootPolicy)
}
itemPolicy, ok := root.at(Keys("items"))
if !ok || len(itemPolicy.recordOrder) != 1 {
t.Fatalf("unrelated item owner disappeared: %#v", itemPolicy)
}
itemRecord, _ := itemPolicy.recordPolicyAt(0)
nested, ok := itemRecord.scope.at(Keys("steps"))
if !ok || !nested.identityPresent || !reflect.DeepEqual(nested.identityFields, []string{"item-id"}) {
t.Fatalf("item-local policy was coupled to root destination: %#v", nested)
}
}
func TestCollectionPolicyProjectionUsesConcretePlacement(t *testing.T) {
first := newCollectionRecordPolicy()
first.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"step"}, identityPresent: true})
second := newCollectionRecordPolicy()
second.scope.set(Keys("steps"), collectionPolicy{explicitMode: sequenceModeAppend, explicitModePresent: true})
var scope collectionPolicyScope
scope.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, first, second))
projected := projectCollectionPolicyScope(scope, Keys("mounted"))
if len(projected) != 3 {
t.Fatalf("projected policies = %#v", projected)
}
want := []string{"/mounted/items", "/mounted/items/0/steps", "/mounted/items/1/steps"}
for index, policy := range projected {
if policy.path.String() != want[index] {
t.Fatalf("projected path[%d] = %q, want %q", index, policy.path.String(), want[index])
}
}
}
func TestCollectionPolicyInheritanceCopiesOwnersAndState(t *testing.T) {
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
original := collectionPolicyWithRecords(collectionPolicy{}, owner)
copy := cloneCollectionPolicyForInheritance(original)
copyOwner := copy.recordOrder[0]
if copyOwner == owner.owner {
t.Fatal("inheritance copy retained the source record owner")
}
copy.recordPolicies[copyOwner].collections["/steps"].policy.identityFields[0] = "changed"
if owner.scope.collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("inheritance copy aliases source policy")
}
}
func TestAcquisitionScopeCloneOwnsCollectionPolicyCopy(t *testing.T) {
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
var scope acquisitionScope
scope.collectionPolicies.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, owner))
cloned := cloneAcquisitionScope(scope)
clonedOwner := cloned.collectionPolicies.collections["/items"].policy.recordOrder[0]
cloned.collectionPolicies.collections["/items"].policy.recordPolicies[clonedOwner].collections["/steps"].policy.identityFields[0] = "changed"
sourceOwner := scope.collectionPolicies.collections["/items"].policy.recordOrder[0]
if scope.collectionPolicies.collections["/items"].policy.recordPolicies[sourceOwner].collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("acquisition scope clone aliases collection policy")
}
if clonedOwner != owner.owner {
t.Fatal("acquisition scope clone changed record ownership")
}
}
func collectionPolicyWithRecords(policy collectionPolicy, records ...collectionRecordPolicy) collectionPolicy {
for _, record := range records {
policy.setRecordPolicy(record)
}
return policy
}
func parsePolicyNode(t *testing.T, input string) *document.Node {
t.Helper()
parsed, diagnostics := parseSource(Bytes("collection-policy-state.yml", []byte(input)), "collection-policy-state", defaultParseLimits())
if len(diagnostics) != 0 {
t.Fatalf("parse policy value: %#v", diagnostics)
}
return parsed.document
}

View File

@@ -0,0 +1,192 @@
package engine
import (
"reflect"
"testing"
)
func TestResolveCollectionPolicyReconcilesDeclarationsBeforeComposition(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: [namespace, id]
mode: append
- destination: [prompts]
key: [id, namespace, id]
- destination: [prompts]
mode: append
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, nil)
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
if !resolved.declarationsPresent || !resolved.identityDeclarationPresent || !resolved.modeDeclarationPresent {
t.Fatalf("declaration presence was lost: %#v", resolved)
}
wantFields := []string{"id", "namespace"}
if !reflect.DeepEqual(resolved.policy.identityFields, wantFields) || !reflect.DeepEqual(resolved.effectiveIdentityFields, wantFields) {
t.Fatalf("effective identity = %#v / %#v, want %#v", resolved.policy.identityFields, resolved.effectiveIdentityFields, wantFields)
}
if !resolved.policy.identityPresent || !resolved.policy.explicitModePresent || resolved.policy.explicitMode != sequenceModeAppend || resolved.operation != sequenceModeAppend {
t.Fatalf("resolved explicit policy = %#v, operation %d", resolved.policy, resolved.operation)
}
if len(resolved.declarations) != 3 || resolved.declarations[0].identityLocation == nil || resolved.declarations[2].location.Line == 0 {
t.Fatalf("declaration locations were not retained: %#v", resolved.declarations)
}
}
func TestResolveCollectionPolicyRejectsConflictingPeersAtBothSites(t *testing.T) {
tests := []struct {
name string
body string
}{
{
name: "identity fields",
body: `configkit:
sequences:
- destination: [prompts]
key: id
- destination: [prompts]
key: name
`,
},
{
name: "explicit modes",
body: `configkit:
sequences:
- destination: [prompts]
mode: append
- destination: [prompts]
mode: replace
`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, test.body, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, nil)
if !reflect.DeepEqual(resolved, collectionPolicyResolution{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("resolution/diagnostics = %#v / %#v", resolved, diagnostics)
}
diagnostic := diagnostics[0]
if diagnostic.Path.String() != "/prompts" || diagnostic.Primary == nil || len(diagnostic.Related) != 1 {
t.Fatalf("conflict does not identify both declaration sites: %#v", diagnostic)
}
if diagnostic.Primary.SourceID != "sequence-declarations" || diagnostic.Related[0].SourceID != "sequence-declarations" || diagnostic.Primary.Line == diagnostic.Related[0].Line {
t.Fatalf("conflict locations = %#v and %#v", diagnostic.Primary, diagnostic.Related)
}
})
}
}
func TestResolveCollectionPolicySelectsCurrentThenRetainedExplicitMode(t *testing.T) {
lower := &collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
modeLocation: &Location{SourceID: "lower", Line: 2},
}
current := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
mode: upsert
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), current, nil, lower)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.explicitMode != sequenceModeUpsert || !resolved.policy.explicitModePresent {
t.Fatalf("current mode did not override retained mode: %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), nil, nil, lower)
if len(diagnostics) != 0 || resolved.operation != sequenceModeAppend || resolved.policy.explicitMode != sequenceModeAppend || !resolved.policy.explicitModePresent {
t.Fatalf("omitted current mode did not retain lower explicit mode: %#v, %#v", resolved, diagnostics)
}
if !reflect.DeepEqual(resolved.effectiveIdentityFields, []string{"id"}) {
t.Fatalf("retained identity = %#v", resolved.effectiveIdentityFields)
}
}
func TestResolveCollectionPolicyInfersWithoutPersistingMode(t *testing.T) {
keyOnly := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: id
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), keyOnly, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.explicitModePresent {
t.Fatalf("identity-based mode inference = %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), nil, []string{"id"}, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.identityPresent || !resolved.applicationIdentityPresent {
t.Fatalf("application identity was not kept distinct from YAML policy: %#v, %#v", resolved, diagnostics)
}
noIdentity := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
`, rootDirectiveContextSource).sequences
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), noIdentity, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeReplace || resolved.policy.explicitModePresent {
t.Fatalf("unkeyed mode inference = %#v, %#v", resolved, diagnostics)
}
explicitDefault := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
mode: replace
`, rootDirectiveContextSource).sequences
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), explicitDefault, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeReplace || !resolved.policy.explicitModePresent {
t.Fatalf("explicit default mode lost its presence: %#v, %#v", resolved, diagnostics)
}
}
func TestResolveCollectionPolicyChecksKeyOnlyAgainstApplicationConstraint(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: [namespace, id]
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, []string{"id", "namespace", "id"}, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || !reflect.DeepEqual(resolved.effectiveIdentityFields, []string{"id", "namespace"}) {
t.Fatalf("equivalent key-only declaration was rejected: %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), declarations, []string{"slug"}, nil)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("conflicting key-only declaration = %#v, %#v", resolved, diagnostics)
}
if diagnostics[0].Primary == nil || diagnostics[0].Path.String() != "/prompts" || len(diagnostics[0].Related) != 0 {
t.Fatalf("application conflict diagnostic = %#v", diagnostics[0])
}
}
func TestResolveCollectionPolicyChecksRetainedIdentityAgainstApplicationConstraint(t *testing.T) {
lower := &collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), nil, []string{"name"}, lower)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/prompts" {
t.Fatalf("retained/application conflict = %#v, %#v", resolved, diagnostics)
}
}
func TestResolveCollectionPolicyRequiresCurrentIdentityToAgreeWithRetainedIdentity(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: name
`, rootDirectiveContextSource).sequences
lower := &collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
identityLocation: &Location{SourceID: "lower", Line: 4},
}
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, lower)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/prompts" {
t.Fatalf("current/retained identity conflict = %#v, %#v", resolved, diagnostics)
}
if diagnostics[0].Primary == nil || len(diagnostics[0].Related) != 1 || diagnostics[0].Related[0].SourceID != "lower" {
t.Fatalf("current/retained conflict locations = %#v", diagnostics[0])
}
}

View File

@@ -0,0 +1,81 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
// A true entry replaces the collection itself as well as its descendants;
// false records ordinary ancestor replacement without selecting a collection
// operation at that exact path.
type collectionReplacementPaths map[string]bool
func (paths collectionReplacementPaths) contains(path Path, includeExact bool) bool {
for length := 0; length <= len(path.segments); length++ {
prefix := Path{segments: path.segments[:length]}
inclusive, found := paths[prefix.String()]
if found && (length < len(path.segments) || includeExact && inclusive) {
return true
}
}
return false
}
// lowerCollectionPoliciesForClaims makes replacement eligibility independent of
// declaration order. It removes only old YAML identity/record state; new pending
// declarations and application constraints are reconciled afterward.
func lowerCollectionPoliciesForClaims(
lower collectionPolicyScope,
claims []compositionClaim,
declarations map[string][]sequenceDeclarationSpec,
application sequencePolicy,
) (collectionPolicyScope, collectionReplacementPaths) {
byPath := make(map[string]compositionClaim, len(claims))
rootPath := ""
if len(claims) != 0 {
rootPath = claims[0].path.String()
}
for _, claim := range claims {
byPath[claim.path.String()] = claim
}
replaced := make(collectionReplacementPaths)
for key, claim := range byPath {
old, hasOld := lower.collections[key]
current := declarations[key]
if claim.kind == document.Mapping && (key == rootPath || !hasOld && len(current) == 0) {
continue // Ordinary mapping combination preserves descendants.
}
mode := sequenceMode(0)
identity := old.policy.identityPresent
for _, declaration := range current {
identity = identity || declaration.identityPresent
if mode == 0 && declaration.modePresent {
mode = declaration.mode
}
}
retainedMode := sequenceMode(0)
if old.policy.explicitModePresent {
retainedMode = old.policy.explicitMode
}
_, registered := application.identityFieldsAt(claim.path)
mode = collectionOperationMode(mode, retainedMode, identity || registered)
if mode == sequenceModeReplace {
replaced[key] = true
} else if claim.kind != document.Mapping && claim.kind != document.Sequence {
replaced[key] = false
}
}
result := cloneCollectionPolicyScope(lower)
for key, entry := range result.collections {
if replaced.contains(entry.destination, true) {
clearCollectionIdentity(&entry.policy)
result.collections[key] = entry
}
}
return result, replaced
}
func clearCollectionIdentity(policy *collectionPolicy) {
policy.identityFields = nil
policy.identityPresent = false
policy.identityLocation = nil
policy.recordOrder = nil
policy.recordPolicies = nil
}

View File

@@ -0,0 +1,111 @@
package engine
import (
"errors"
"testing"
)
func TestSourceReplacementRetiresIdentityForEveryValueKind(t *testing.T) {
t.Parallel()
for _, value := range []string{"null", "42", "{}", "[{name: new}]"} {
t.Run(value, func(t *testing.T) {
lower := "configkit:\n sequences: [{destination: [items], key: id}]\nitems: [{id: old}]\n"
main := "configkit:\n sequences: [{destination: [items], key: name, mode: replace}]\nitems: " + value + "\n"
loaded, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(lower))}}}},
})
if err != nil || loaded == nil {
t.Fatalf("replacement retained obsolete identity: %v", err)
}
})
}
}
func TestAncestorRemovalCanEstablishPendingReplacementIdentity(t *testing.T) {
t.Parallel()
for _, inheritance := range []bool{false, true} {
name := "tiers"
if inheritance {
name = "inheritance"
}
t.Run(name, func(t *testing.T) {
var loaded *testResult[map[string]any]
var err error
if inheritance {
loaded, err = testLoad[map[string]any](Bytes("source", []byte(`configkit:
sequences:
- {destination: [base, group, items], key: id}
- {destination: [child, group, items], key: name}
base: {group: {items: [{id: old}]}}
child:
extends: base
group: null
grandchild:
extends: child
group: {items: [{name: new}]}
`)))
} else {
loaded, err = testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte("group: {items: [{name: new}]}\n"))},
underlays: []loadGroup{
{inputs: []loadInput{{source: Bytes("base", []byte("configkit:\n sequences: [{destination: [group, items], key: id}]\ngroup: {items: [{id: old}]}\n"))}}},
{inputs: []loadInput{{source: Bytes("remove", []byte("configkit:\n sequences: [{destination: [group, items], key: name}]\ngroup: null\n"))}}},
},
})
}
if err != nil {
t.Fatal(err)
}
value := loaded.Config
if inheritance {
value = value["grandchild"].(map[string]any)
}
items := value["group"].(map[string]any)["items"].([]any)
if len(items) != 1 || items[0].(map[string]any)["name"] != "new" {
t.Fatalf("reintroduced collection = %#v", items)
}
})
}
}
func TestInheritanceAncestorRetirementDoesNotResurrectOldIdentity(t *testing.T) {
t.Parallel()
_, err := testLoad[map[string]any](Bytes("source", []byte(`configkit:
sequences: [{destination: [base, group, items], key: id}]
base: {group: {items: [{id: old}]}}
child:
extends: base
group: null
grandchild:
extends: child
group: {items: [{name: new}]}
`)))
if err != nil {
t.Fatal(err)
}
}
func TestReplacementRetirementPreservesApplicationConstraintsAndRetainedKeys(t *testing.T) {
t.Parallel()
for _, withValue := range []bool{false, true} {
main := "configkit:\n sequences: [{destination: [items], key: name, mode: replace}]\n"
if withValue {
main += "items: null\n"
}
definition := Definition[map[string]any]{}
if withValue {
definition.sequenceIdentityRegistrations = []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"},
}}
}
_, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))}, definition: definition,
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences: [{destination: [items], key: id}]\nitems: [{id: old}]\n"))}}}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != CodeSequenceConflict {
t.Fatalf("retirement bypassed identity agreement: %v", err)
}
}
}

View File

@@ -1,20 +1,26 @@
package engine package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" import (
"maps"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
// composedDocument is the private handoff between source assembly and later // composedDocument is the private handoff between source assembly and later
// precedence layers. Claims remain ordered by source and declaration traversal; // precedence layers. Claims remain ordered by source and declaration traversal;
// origins retain every source that contributes to a composite mapping. // origins retain every source that contributes to a composite mapping.
type composedDocument struct { type composedDocument struct {
root *document.Node root *document.Node
acquisition []acquisitionScope acquisition []acquisitionScope
claims []compositionClaim importGraph importInclusionGraph
sequenceRanges []sequenceRange collectionPolicies collectionPolicyScope
origins map[string]Origin claims []compositionClaim
overlays []overlayRecord sequenceRanges []sequenceRange
applications []historyApplication origins map[string]Origin
inputBytes int64 overlays []overlayRecord
inputNodes int applications []historyApplication
inputBytes int64
inputNodes int
} }
type sequenceHistoryMode uint8 type sequenceHistoryMode uint8
@@ -22,13 +28,16 @@ type sequenceHistoryMode uint8
const ( const (
sequenceHistoryReplace sequenceHistoryMode = iota + 1 sequenceHistoryReplace sequenceHistoryMode = iota + 1
sequenceHistoryAppend sequenceHistoryAppend
sequenceHistoryMapped
) )
type sequenceHistoryContribution struct { type sequenceHistoryContribution struct {
path Path path Path
mode sequenceHistoryMode mode sequenceHistoryMode
offset int offset int
length int length int
items []sequenceItemProjection
replacements []sequenceItemReplacement
} }
// assemblyAccumulator owns one mutable working document. Acquired trees held by // assemblyAccumulator owns one mutable working document. Acquired trees held by
@@ -50,9 +59,11 @@ func cloneComposedDocument(base *composedDocument) *composedDocument {
} }
return &composedDocument{ return &composedDocument{
root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims), root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims),
acquisition: cloneAcquisitionScopes(base.acquisition), acquisition: cloneAcquisitionScopes(base.acquisition),
sequenceRanges: cloneSequenceRanges(base.sequenceRanges), importGraph: base.importGraph.clone(),
origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays), collectionPolicies: cloneCollectionPolicyScope(base.collectionPolicies),
sequenceRanges: cloneSequenceRanges(base.sequenceRanges),
origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays),
applications: cloneHistoryApplications(base.applications), applications: cloneHistoryApplications(base.applications),
inputBytes: base.inputBytes, inputNodes: base.inputNodes, inputBytes: base.inputBytes, inputNodes: base.inputNodes,
} }
@@ -78,6 +89,53 @@ type sequenceRange struct {
path Path path Path
start int start int
end int end int
items []sequenceItemProjection
projection sequenceItemProjectionIndex
}
func (assigned sequenceRange) outputForInput(inputIndex int) (int, bool) {
if assigned.projection.outputByInput != nil {
return assigned.projection.outputForInput(inputIndex)
}
if assigned.items != nil {
output, found := 0, false
for _, item := range assigned.items {
if item.inputIndex == inputIndex && inputIndex >= 0 && item.outputIndex >= 0 {
output, found = item.outputIndex, true
}
}
return output, found
}
if inputIndex < 0 || inputIndex >= assigned.end-assigned.start {
return 0, false
}
return assigned.start + inputIndex, true
}
func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) {
if assigned.projection.inputByOutput != nil {
return assigned.projection.inputForOutput(outputIndex)
}
if assigned.items != nil {
input, found := 0, false
for _, item := range assigned.items {
if item.outputIndex == outputIndex && outputIndex >= 0 && item.inputIndex >= 0 {
input, found = item.inputIndex, true
}
}
return input, found
}
if outputIndex < assigned.start || outputIndex >= assigned.end {
return 0, false
}
return outputIndex - assigned.start, true
}
func indexSequenceRange(assigned sequenceRange) sequenceRange {
if assigned.items != nil && assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil {
assigned.projection = newSequenceItemProjectionIndex(assigned.items)
}
return assigned
} }
// composeDocuments recursively joins disjoint mapping entries. Source order // composeDocuments recursively joins disjoint mapping entries. Source order
@@ -91,6 +149,17 @@ func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic
// sequence rules when policy contains registrations. The ordinary adapter above // sequence rules when policy contains registrations. The ordinary adapter above
// deliberately preserves the existing unregistered behavior for current callers. // deliberately preserves the existing unregistered behavior for current callers.
func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) (*composedDocument, []Diagnostic) { func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) (*composedDocument, []Diagnostic) {
return composeDocumentsWithCollectionPolicy(policy, collectionPolicyScope{}, nil, inputs...)
}
// composeDocumentsWithCollectionPolicy resolves collection declarations and
// eligible lower policy before strictly composing completed peer documents.
func composeDocumentsWithCollectionPolicy(
policy sequencePolicy,
lower collectionPolicyScope,
declarations []sequenceDeclarationSpec,
inputs ...*document.Node,
) (*composedDocument, []Diagnostic) {
if len(inputs) == 0 { if len(inputs) == 0 {
return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires at least one document", nil)} return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires at least one document", nil)}
} }
@@ -108,6 +177,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
collected.append(input, Path{}, sourceIndex) collected.append(input, Path{}, sourceIndex)
} }
identityFieldsByPath, collectionPolicies, diagnostics := resolveStrictCollectionPolicies(
collected.ordered, policy, lower, declarations,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
checked := make(map[string]struct{}, len(collected.byPath)) checked := make(map[string]struct{}, len(collected.byPath))
for _, claim := range collected.ordered { for _, claim := range collected.ordered {
pointer := claim.path.String() pointer := claim.path.String()
@@ -116,13 +192,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
} }
checked[pointer] = struct{}{} checked[pointer] = struct{}{}
claims := collected.byPath[pointer] claims := collected.byPath[pointer]
if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && policy.registered(claim.path)) { if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && len(identityFieldsByPath[pointer]) != 0) {
continue continue
} }
return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)} return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)}
} }
validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy) validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, identityFieldsByPath)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
return nil, diagnostics return nil, diagnostics
} }
@@ -130,7 +206,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
root := document.Clone(inputs[0]) root := document.Clone(inputs[0])
index := new(mappingIndex) index := new(mappingIndex)
for sourceIndex, input := range inputs[1:] { for sourceIndex, input := range inputs[1:] {
mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, policy, validated) mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, identityFieldsByPath, validated)
} }
applications := make([]historyApplication, 0, len(inputs)) applications := make([]historyApplication, 0, len(inputs))
seenSequences := make(map[string]bool) seenSequences := make(map[string]bool)
@@ -157,11 +233,12 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
origins := projectedCompositionOrigins(inputs, ranges) origins := projectedCompositionOrigins(inputs, ranges)
completeCompositionOrigins(root, Path{}, origins) completeCompositionOrigins(root, Path{}, origins)
return &composedDocument{ return &composedDocument{
root: root, root: root,
claims: cloneCompositionClaims(collected.ordered), collectionPolicies: collectionPolicies,
sequenceRanges: cloneSequenceRanges(ranges), claims: cloneCompositionClaims(collected.ordered),
origins: origins, sequenceRanges: cloneSequenceRanges(ranges),
applications: applications, origins: origins,
applications: applications,
}, nil }, nil
} }
@@ -217,71 +294,128 @@ type validatedSequence struct {
items []*document.Node items []*document.Node
} }
func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) { func resolveStrictCollectionPolicies(
claims []compositionClaim,
applicationPolicy sequencePolicy,
lower collectionPolicyScope,
declarations []sequenceDeclarationSpec,
) (map[string][]string, collectionPolicyScope, []Diagnostic) {
paths := make([]Path, 0, len(claims)+len(declarations))
seen := make(map[string]bool, len(claims)+len(declarations))
valuePresent := make(map[string]bool)
declarationsByPath := make(map[string][]sequenceDeclarationSpec, len(declarations))
for _, claim := range claims {
if len(lower.collections) != 0 {
valuePresent[claim.path.String()] = true
}
if claim.kind != document.Sequence {
continue
}
key := claim.path.String()
if !seen[key] {
seen[key] = true
paths = append(paths, claim.path.clone())
}
}
for _, declaration := range declarations {
key := declaration.destination.String()
declarationsByPath[key] = append(declarationsByPath[key], declaration)
if !seen[key] {
seen[key] = true
paths = append(paths, declaration.destination.clone())
}
}
identityFieldsByPath := make(map[string][]string)
effectiveLower := lower
if len(lower.collections) != 0 {
effectiveLower, _ = lowerCollectionPoliciesForClaims(lower, claims, declarationsByPath, applicationPolicy)
}
collectionPolicies := cloneCollectionPolicyScope(effectiveLower)
var diagnostics []Diagnostic
for _, path := range paths {
key := path.String()
currentDeclarations := declarationsByPath[key]
lowerEntry, hasLower := effectiveLower.collections[key]
var lowerPolicy *collectionPolicy
if hasLower {
copy := cloneCollectionPolicy(lowerEntry.policy)
lowerPolicy = &copy
}
applicationIdentity, _ := applicationPolicy.identityFieldsAt(path)
resolution, found := resolveCollectionPolicy(path, currentDeclarations, applicationIdentity, lowerPolicy)
if len(found) != 0 {
diagnostics = append(diagnostics, found...)
continue
}
if len(resolution.effectiveIdentityFields) != 0 {
identityFieldsByPath[key] = cloneStrings(resolution.effectiveIdentityFields)
}
policyForPath := resolution.policy
if hasLower && !valuePresent[key] {
// A declaration without a new value changes policy metadata but
// leaves retained record ownership untouched.
policyForPath = cloneCollectionPolicy(lowerEntry.policy)
policyForPath.identityFields = cloneStrings(resolution.policy.identityFields)
policyForPath.identityPresent = resolution.policy.identityPresent
policyForPath.identityLocation = cloneLocation(resolution.policy.identityLocation)
policyForPath.explicitMode = resolution.policy.explicitMode
policyForPath.explicitModePresent = resolution.policy.explicitModePresent
policyForPath.modeLocation = cloneLocation(resolution.policy.modeLocation)
}
if collectionPolicies.collections == nil {
collectionPolicies.collections = make(map[string]collectionPolicyEntry)
}
collectionPolicies.collections[key] = collectionPolicyEntry{destination: path.clone(), policy: policyForPath}
}
return identityFieldsByPath, collectionPolicies, canonicalDiagnostics(diagnostics)
}
func validateRegisteredSequences(claims []compositionClaim, identityFieldsByPath map[string][]string) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
itemsByPath := make(map[string]map[int]validatedSequence) itemsByPath := make(map[string]map[int]validatedSequence)
identitiesByPath := make(map[string]map[string]Location) identitiesByPath := make(map[string]map[string]document.Location)
lengthsByPath := make(map[string]int) lengthsByPath := make(map[string]int)
var ranges []sequenceRange var ranges []sequenceRange
for _, claim := range claims { for _, claim := range claims {
if claim.kind != document.Sequence { if claim.kind != document.Sequence {
continue continue
} }
identityField, registered := policy.identityFields[claim.path.String()] identityFields := identityFieldsByPath[claim.path.String()]
if !registered { if len(identityFields) == 0 {
continue continue
} }
node := claim.node node := claim.node
pathKey := claim.path.String() pathKey := claim.path.String()
identities := identitiesByPath[pathKey] identities := identitiesByPath[pathKey]
if identities == nil { if identities == nil {
identities = make(map[string]Location, len(node.Sequence)) identities = make(map[string]document.Location, len(node.Sequence))
identitiesByPath[pathKey] = identities identitiesByPath[pathKey] = identities
} }
start := lengthsByPath[pathKey] start := lengthsByPath[pathKey]
contribution := validatedSequence{items: node.Sequence} contribution := validatedSequence{items: node.Sequence}
for itemIndex, item := range node.Sequence { records, diagnostics := validateNormalizedSequenceIdentityFrom(node, claim.path, identityFields, sequenceIdentityDiagnosticCodes{
itemPath := claim.path.Index(start + itemIndex) invalid: CodeComposeInvalid, conflict: CodeComposeConflict,
if item == nil || item.Kind != document.Mapping { }, start, sequenceStructuralRecord)
location := (*Location)(nil) if len(diagnostics) != 0 {
if item != nil { return nil, nil, diagnostics
converted := publicLocation(item.Location) }
location = &converted byIndex := make([]sequenceIdentityRecord, len(node.Sequence))
} for _, record := range records {
return nil, nil, []Diagnostic{{ byIndex[record.index] = record
Code: CodeComposeInvalid, Severity: SeverityError, }
Path: itemPath, Message: "registered sequence items must be mappings", Primary: location, for _, record := range byIndex {
}} if first, duplicate := identities[record.tuple]; duplicate {
} current := publicLocation(record.location)
fieldIndex := documentEntryIndex(item.Mapping, identityField)
if fieldIndex < 0 {
location := publicLocation(item.Location)
return nil, nil, []Diagnostic{{
Code: CodeComposeInvalid, Severity: SeverityError,
Path: itemPath, Message: "registered sequence item is missing its identity field",
Primary: &location,
}}
}
field := item.Mapping[fieldIndex].Value
if field == nil || field.Kind != document.String {
location := publicLocation(item.Mapping[fieldIndex].ValueLocation)
if field != nil {
location = publicLocation(field.Location)
}
return nil, nil, []Diagnostic{{
Code: CodeComposeInvalid, Severity: SeverityError,
Path: itemPath.Key(identityField), Message: "registered sequence identity must be a string",
Primary: &location,
}}
}
if first, duplicate := identities[field.Text]; duplicate {
current := publicLocation(field.Location)
return nil, nil, []Diagnostic{{ return nil, nil, []Diagnostic{{
Code: CodeComposeConflict, Severity: SeverityError, Code: CodeComposeConflict, Severity: SeverityError,
Path: claim.path.clone(), Message: "registered sequence contains duplicate identities", Path: claim.path.clone(), Message: "registered sequence contains duplicate identities",
Primary: &first, Related: []Location{current}, Primary: ptrLocation(publicLocation(first)), Related: []Location{current},
}} }}
} }
identities[field.Text] = publicLocation(field.Location) identities[record.tuple] = record.location
} }
if itemsByPath[pathKey] == nil { if itemsByPath[pathKey] == nil {
itemsByPath[pathKey] = make(map[int]validatedSequence) itemsByPath[pathKey] = make(map[int]validatedSequence)
@@ -295,7 +429,7 @@ func validateRegisteredSequences(claims []compositionClaim, policy sequencePolic
return itemsByPath, ranges, nil return itemsByPath, ranges, nil
} }
func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) { func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
for _, incomingEntry := range incoming.Mapping { for _, incomingEntry := range incoming.Mapping {
entryPath := path.Key(incomingEntry.Key) entryPath := path.Key(incomingEntry.Key)
position := index.find(target, incomingEntry.Key) position := index.find(target, incomingEntry.Key)
@@ -306,7 +440,7 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
continue continue
} }
current := target.Mapping[position].Value current := target.Mapping[position].Value
if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && policy.registered(entryPath) { if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && len(identityFieldsByPath[entryPath.String()]) != 0 {
contribution := validated[entryPath.String()][sourceIndex] contribution := validated[entryPath.String()][sourceIndex]
if contribution.items != nil { if contribution.items != nil {
for _, item := range contribution.items { for _, item := range contribution.items {
@@ -315,20 +449,15 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
} }
continue continue
} }
mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, policy, validated) mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, identityFieldsByPath, validated)
} }
} }
func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) { func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
if target.Kind != document.Mapping || incoming.Kind != document.Mapping { if target.Kind != document.Mapping || incoming.Kind != document.Mapping {
return return
} }
mergeComposedMappings(target, incoming, path, index, sourceIndex, policy, validated) mergeComposedMappings(target, incoming, path, index, sourceIndex, identityFieldsByPath, validated)
}
func (p sequencePolicy) registered(path Path) bool {
_, ok := p.identityFields[path.String()]
return ok
} }
func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange { func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {
@@ -336,6 +465,11 @@ func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {
for i, assigned := range ranges { for i, assigned := range ranges {
cloned[i] = assigned cloned[i] = assigned
cloned[i].path = assigned.path.clone() cloned[i].path = assigned.path.clone()
if assigned.items != nil {
cloned[i].items = append([]sequenceItemProjection{}, assigned.items...)
}
cloned[i].projection.outputByInput = maps.Clone(assigned.projection.outputByInput)
cloned[i].projection.inputByOutput = maps.Clone(assigned.projection.inputByOutput)
} }
return cloned return cloned
} }
@@ -375,12 +509,28 @@ func documentEntryIndex(entries []document.Entry, key string) int {
func projectedCompositionOrigins(inputs []*document.Node, ranges []sequenceRange) map[string]Origin { func projectedCompositionOrigins(inputs []*document.Node, ranges []sequenceRange) map[string]Origin {
origins := make(map[string]Origin) origins := make(map[string]Origin)
rangeBySourcePath := make(map[sequenceRangeKey]int, len(ranges)) sequencesBySource := make([][]sequenceHistoryContribution, len(inputs))
for index, assigned := range ranges { for _, assigned := range ranges {
rangeBySourcePath[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = index contribution := sequenceHistoryContribution{
path: assigned.path.clone(), mode: sequenceHistoryAppend,
offset: assigned.start, length: assigned.end - assigned.start,
}
if assigned.items != nil {
contribution.mode = sequenceHistoryMapped
contribution.items = assigned.items
}
sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], contribution)
} }
for sourceIndex, input := range inputs { for sourceIndex, input := range inputs {
collectProjectedCompositionOrigins(input, Path{}, Path{}, sourceIndex, ranges, rangeBySourcePath, origins) application := historyApplication{value: input, sequences: sequencesBySource[sourceIndex]}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
walkHistoryApplication(application, func(_, effective Path, node *document.Node) {
pointer := effective.String()
origin := origins[pointer]
origin.Layer = LayerBase
origin.Locations = append(origin.Locations, publicLocation(node.Location))
origins[pointer] = origin
})
} }
return origins return origins
} }
@@ -398,39 +548,12 @@ func indexSequenceRanges(ranges []sequenceRange) sequenceRangeIndex {
} }
index := make(sequenceRangeIndex, len(ranges)) index := make(sequenceRangeIndex, len(ranges))
for _, assigned := range ranges { for _, assigned := range ranges {
assigned = indexSequenceRange(assigned)
index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned
} }
return index return index
} }
// Keep local paths for matching Stage 2 ranges and effective paths for emitted
// evidence. The paths are transformed segment by segment; pointer strings are
// only used as map keys after projection is complete.
func collectProjectedCompositionOrigins(node *document.Node, localPath, effectivePath Path, sourceIndex int, ranges []sequenceRange, rangeBySourcePath map[sequenceRangeKey]int, origins map[string]Origin) {
pointer := effectivePath.String()
origin := origins[pointer]
origin.Layer = LayerBase
origin.Locations = append(origin.Locations, publicLocation(node.Location))
origins[pointer] = origin
switch node.Kind {
case document.Mapping:
for _, entry := range node.Mapping {
collectProjectedCompositionOrigins(entry.Value, localPath.Key(entry.Key), effectivePath.Key(entry.Key), sourceIndex, ranges, rangeBySourcePath, origins)
}
case document.Sequence:
start := 0
if rangeIndex, ok := rangeBySourcePath[sequenceRangeKey{sourceIndex: sourceIndex, path: localPath.String()}]; ok {
start = ranges[rangeIndex].start
}
for index, child := range node.Sequence {
localChild := localPath.Index(index)
effectiveChild := effectivePath.Index(start + index)
collectProjectedCompositionOrigins(child, localChild, effectiveChild, sourceIndex, ranges, rangeBySourcePath, origins)
}
}
}
func applyCompositionOrigins(decoded *decodedConfiguration, origins map[string]Origin) { func applyCompositionOrigins(decoded *decodedConfiguration, origins map[string]Origin) {
for pointer, origin := range origins { for pointer, origin := range origins {
if _, present := decoded.presence[pointer]; present { if _, present := decoded.presence[pointer]; present {

View File

@@ -10,17 +10,20 @@ import (
) )
func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) { func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) {
policy := sequencePolicy{identityFields: make(map[string]string)} registrations := make([]sequenceIdentityRegistration, 0, 128)
var inputs []*document.Node var inputs []*document.Node
for source := 0; source < 3; source++ { for source := 0; source < 3; source++ {
var text strings.Builder var text strings.Builder
for collection := 0; collection < 128; collection++ { for collection := 0; collection < 128; collection++ {
key := fmt.Sprintf("records%d", collection) key := fmt.Sprintf("records%d", collection)
policy.identityFields[Keys(key).String()] = "id" registrations = append(registrations, sequenceIdentityRegistration{
selector: sequenceIdentitySelector{literalSequenceSelector(key)}, identityFields: []string{"id"},
})
fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source) fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source)
} }
inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String())) inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String()))
} }
policy := mustCompileSequenceIdentityPolicy(t, registrations)
composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...) composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
t.Fatal(diagnostics) t.Fatal(diagnostics)
@@ -242,7 +245,7 @@ func TestComposeDocumentsAppendsRegisteredSequencesAndReturnsRanges(t *testing.T
first := parseCompositionDocument(t, "source-1", "records: [{id: '', left: 1}, {id: ' spaced ', right: 1}]\nother: [one]\n") first := parseCompositionDocument(t, "source-1", "records: [{id: '', left: 1}, {id: ' spaced ', right: 1}]\nother: [one]\n")
second := parseCompositionDocument(t, "source-2", "records: [{id: ' ', text: second}]\n") second := parseCompositionDocument(t, "source-2", "records: [{id: ' ', text: second}]\n")
firstBefore, secondBefore := document.Clone(first), document.Clone(second) firstBefore, secondBefore := document.Clone(first), document.Clone(second)
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}} policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second) composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
@@ -287,10 +290,10 @@ escaped/~:
name: escaped-second-name name: escaped-second-name
`) `)
firstBefore, secondBefore := document.Clone(first), document.Clone(second) firstBefore, secondBefore := document.Clone(first), document.Clone(second)
policy := sequencePolicy{identityFields: map[string]string{ policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
Keys("mounted", "records").String(): "id", {selector: sequenceIdentitySelector{literalSequenceSelector("mounted"), literalSequenceSelector("records")}, identityFields: []string{"id"}},
Keys("escaped/~", "7").String(): "id", {selector: sequenceIdentitySelector{literalSequenceSelector("escaped/~"), literalSequenceSelector("7")}, identityFields: []string{"id"}},
}} })
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second) composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
@@ -357,6 +360,78 @@ escaped/~:
} }
} }
func TestProjectedCompositionOriginsUseNoncontiguousItemMappings(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
label: equal
- id: middle
- id: last
nested: {old: lower}
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
nested: {new: higher}
- id: first
label: equal
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher, sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
inputs := []*document.Node{lowerRoot, higherRoot}
ranges := []sequenceRange{
{
sourceIndex: 0, path: Keys("records"), start: 0, end: 3,
items: combined.lowerItems,
}, {
sourceIndex: 1, path: Keys("records"), start: 0, end: 3,
items: combined.higherItems,
},
}
origins := projectedCompositionOrigins(inputs, ranges)
if got, want := sequenceIdentityValues(combined.value, "id"), []string{"first", "middle", "last", "added"}; !reflect.DeepEqual(got, want) {
t.Fatalf("combined IDs = %#v, want %#v", got, want)
}
for path, want := range map[string][]string{
"/records/0/label": {"higher"},
"/records/1/id": {"lower"},
"/records/2/nested/new": {"higher"},
"/records/3/id": {"higher"},
"/records/0/nested/new": nil,
"/records/2/nested/old": nil,
} {
origin := origins[path]
var got []string
for _, location := range origin.Locations {
got = append(got, location.SourceID)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("origin sources at %s = %#v, want %#v", path, got, want)
}
}
}
func TestCloneSequenceRangesPreservesPreparedProjection(t *testing.T) {
ranges := []sequenceRange{{
start: 0, end: 1,
projection: newSequenceItemProjectionIndex([]sequenceItemProjection{{inputIndex: 0, outputIndex: 4}}),
}}
cloned := cloneSequenceRanges(ranges)
if output, ok := cloned[0].outputForInput(0); !ok || output != 4 {
t.Fatalf("cloned projection = %d, %v; want output index 4", output, ok)
}
cloned[0].projection.outputByInput[0] = 9
if output, ok := ranges[0].outputForInput(0); !ok || output != 4 {
t.Fatalf("cloned projection aliases original: %d, %v", output, ok)
}
}
func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) { func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -365,15 +440,12 @@ func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
wantCode Code wantCode Code
}{ }{
{name: "nonmapping item", body: "records: [plain]\n", path: "/records/0", wantCode: CodeComposeInvalid}, {name: "nonmapping item", body: "records: [plain]\n", path: "/records/0", wantCode: CodeComposeInvalid},
{name: "missing identity", body: "records: [{name: plain}]\n", path: "/records/0", wantCode: CodeComposeInvalid}, {name: "missing identity", body: "records: [{name: plain}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "null identity", body: "records: [{id: null}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "numeric identity", body: "records: [{id: 12}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "boolean identity", body: "records: [{id: false}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy( _, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", test.body), parseCompositionDocument(t, "source-1", test.body),
) )
if len(diagnostics) != 1 || diagnostics[0].Code != test.wantCode || diagnostics[0].Path.String() != test.path { if len(diagnostics) != 1 || diagnostics[0].Code != test.wantCode || diagnostics[0].Path.String() != test.path {
@@ -386,14 +458,138 @@ func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
} }
} }
func TestComposeDocumentsUsesLowerIdentityForStrictPeers(t *testing.T) {
path := Keys("records")
lower := collectionPolicyScope{collections: map[string]collectionPolicyEntry{
path.String(): {
destination: path,
policy: collectionPolicy{identityFields: []string{"id"}, identityPresent: true},
},
}}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, nil,
parseCompositionDocument(t, "source-1", "records: [{id: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("compose strict peers with lower identity: %#v", diagnostics)
}
value, present := documentAtPath(composed.root, path)
if !present || value.Kind != document.Sequence || len(value.Sequence) != 2 {
t.Fatalf("composed records = %#v, %v; want two strict peer records", value, present)
}
resolved, present := composed.collectionPolicies.collections[path.String()]
if !present || !resolved.policy.identityPresent || !reflect.DeepEqual(resolved.policy.identityFields, []string{"id"}) {
t.Fatalf("composed collection policy = %#v, %v; want retained lower identity", resolved, present)
}
clone := cloneComposedDocument(composed)
resolved.policy.identityFields[0] = "mutated"
if !reflect.DeepEqual(clone.collectionPolicies.collections[path.String()].policy.identityFields, []string{"id"}) {
t.Fatal("cloned composition shares mutable collection identity fields")
}
}
func TestComposeDocumentsResolvesDeclarationsBeforePeerComposition(t *testing.T) {
path := Keys("records")
declaration := func(source string, line int, fields ...string) sequenceDeclarationSpec {
location := Location{SourceID: source, Line: line}
return sequenceDeclarationSpec{
destination: path, identityFields: fields, identityPresent: true,
identityLocation: &location,
}
}
_, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "id"), declaration("source-2", 4, "name")},
parseCompositionDocument(t, "source-1", "left: true\n"),
parseCompositionDocument(t, "source-2", "right: true\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("conflicting policy-only declarations = %#v", diagnostics)
}
if diagnostics[0].Primary == nil || len(diagnostics[0].Related) != 1 {
t.Fatalf("declaration conflict lacks both locations: %#v", diagnostics[0])
}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "tenant", "id"), declaration("source-2", 4, "id", "tenant")},
parseCompositionDocument(t, "source-1", "records: [{tenant: true, id: null}, {tenant: \"alpha\", id: 1}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: 1.0, tenant: \"beta\"}, {id: \"1\", tenant: \"alpha\"}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("compose normalized compound identities: %#v", diagnostics)
}
value, present := documentAtPath(composed.root, path)
if !present || len(value.Sequence) != 4 {
t.Fatalf("compound identity records = %#v, %v; want four distinct records", value, present)
}
_, diagnostics = composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "id")},
parseCompositionDocument(t, "source-1", "records: [{id: 1}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: 1.0}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("equivalent numeric peer identities = %#v", diagnostics)
}
}
func TestComposeDocumentsReplacementDoesNotBorrowLowerYAMLIdentity(t *testing.T) {
path := Keys("records")
lower := collectionPolicyScope{collections: map[string]collectionPolicyEntry{
path.String(): {
destination: path,
policy: collectionPolicy{identityFields: []string{"id"}, identityPresent: true},
},
}}
declaration := sequenceDeclarationSpec{destination: path, mode: sequenceModeReplace, modePresent: true}
_, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, []sequenceDeclarationSpec{declaration},
parseCompositionDocument(t, "source-1", "records: [{name: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: second}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("replacement peers borrowing lower identity = %#v", diagnostics)
}
identityReplacement := sequenceDeclarationSpec{
destination: path, identityFields: []string{"name"}, identityPresent: true,
mode: sequenceModeReplace, modePresent: true,
}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, []sequenceDeclarationSpec{identityReplacement},
parseCompositionDocument(t, "source-1", "records: [{name: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("replacement with a new identity declaration: %#v", diagnostics)
}
resolved := composed.collectionPolicies.collections[path.String()].policy
if !reflect.DeepEqual(resolved.identityFields, []string{"name"}) || !resolved.identityPresent {
t.Fatalf("replacement collection identity = %#v; want name", resolved)
}
_, diagnostics = composeDocumentsWithCollectionPolicy(
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}), lower,
[]sequenceDeclarationSpec{declaration},
parseCompositionDocument(t, "source-1", "records: [{name: replacement}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/0/id" {
t.Fatalf("replacement ignored application identity constraint = %#v", diagnostics)
}
}
func TestComposeDocumentsReportsAssignedPathForMalformedRegisteredSequenceContribution(t *testing.T) { func TestComposeDocumentsReportsAssignedPathForMalformedRegisteredSequenceContribution(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy( _, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "records: [{id: first}]\n"), parseCompositionDocument(t, "source-1", "records: [{id: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: malformed}]\n"), parseCompositionDocument(t, "source-2", "records: [{name: malformed}]\n"),
) )
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/1" { if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/1/id" {
t.Fatalf("diagnostics = %#v, want invalid item at /records/1", diagnostics) t.Fatalf("diagnostics = %#v, want invalid identity field at /records/1/id", diagnostics)
} }
if diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "source-2" { if diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "source-2" {
t.Fatalf("malformed item location = %#v, want source-2", diagnostics[0].Primary) t.Fatalf("malformed item location = %#v, want source-2", diagnostics[0].Primary)
@@ -420,7 +616,7 @@ func TestComposeDocumentsRejectsRegisteredDuplicateIdentities(t *testing.T) {
inputs = append(inputs, parseCompositionDocument(t, "source-2", test.second)) inputs = append(inputs, parseCompositionDocument(t, "source-2", test.second))
} }
_, diagnostics := composeDocumentsWithPolicy( _, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, inputs..., mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}), inputs...,
) )
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" { if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("diagnostics = %#v, want duplicate conflict at /records", diagnostics) t.Fatalf("diagnostics = %#v, want duplicate conflict at /records", diagnostics)
@@ -438,9 +634,12 @@ func TestComposeDocumentsRejectsRegisteredDuplicateIdentities(t *testing.T) {
} }
func TestComposeDocumentsSkipsMissingAndNonSequenceRegistrations(t *testing.T) { func TestComposeDocumentsSkipsMissingAndNonSequenceRegistrations(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{ policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
"/absent": "id", "/not-a-sequence": "id", "/catalog": "id", "/catalog/records": "id", {selector: sequenceIdentitySelector{literalSequenceSelector("absent")}, identityFields: []string{"id"}},
}} {selector: sequenceIdentitySelector{literalSequenceSelector("not-a-sequence")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("catalog")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("catalog"), literalSequenceSelector("records")}, identityFields: []string{"id"}},
})
composed, diagnostics := composeDocumentsWithPolicy(policy, parseCompositionDocument(t, "source-1", ` composed, diagnostics := composeDocumentsWithPolicy(policy, parseCompositionDocument(t, "source-1", `
not-a-sequence: null not-a-sequence: null
catalog: catalog:
@@ -454,7 +653,7 @@ catalog:
} }
composed, diagnostics = composeDocumentsWithPolicy( composed, diagnostics = composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "records: []\n"), parseCompositionDocument(t, "source-1", "records: []\n"),
parseCompositionDocument(t, "source-2", "records: []\n"), parseCompositionDocument(t, "source-2", "records: []\n"),
) )
@@ -471,7 +670,7 @@ catalog:
func TestComposeDocumentsKeepsUnregisteredSequenceConflicts(t *testing.T) { func TestComposeDocumentsKeepsUnregisteredSequenceConflicts(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy( _, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/registered": "id"}}, mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("registered")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "unregistered: [one]\n"), parseCompositionDocument(t, "source-1", "unregistered: [one]\n"),
parseCompositionDocument(t, "source-2", "unregistered: [two]\n"), parseCompositionDocument(t, "source-2", "unregistered: [two]\n"),
) )
@@ -491,7 +690,7 @@ func TestComposeDocumentsKeepsNonSequenceRegisteredOverlapsConflicting(t *testin
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy( _, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", test.first), parseCompositionDocument(t, "source-1", test.first),
parseCompositionDocument(t, "source-2", test.second), parseCompositionDocument(t, "source-2", test.second),
) )

View File

@@ -0,0 +1,77 @@
package engine
import (
"reflect"
"testing"
)
func TestImportAndInheritanceIntegrationsUseEquivalentReadySchedules(t *testing.T) {
names := []string{"a-base", "b-independent", "z-dependent"}
keys := make(map[string]importNodeKey, len(names))
contributions := make([]acquiredContribution, len(names))
for index, name := range names {
key := importNodeKey{
namespaceKind: "host", namespaceAnchor: "/shared",
resourceKind: "file", resourcePath: "/" + name,
}
keys[name] = key
contributions[index] = acquiredContribution{
identity: key,
source: SourceInfo{ID: "source-" + name, Path: name},
}
}
var want []string
for _, reverse := range []bool{false, true} {
graph := newImportInclusionGraph()
orderedNames := append([]string(nil), names...)
if reverse {
orderedNames[0], orderedNames[2] = orderedNames[2], orderedNames[0]
}
for _, name := range orderedNames {
graph.addNode(importGraphNode{key: keys[name], sourceID: "source-" + name, role: importRoleMapping})
}
graph.addEdge(keys["z-dependent"], keys["a-base"], Location{})
scheduled := scheduleContributionSlots(contributions, []int{0, 1, 2}, graph)
got := make([]string, len(scheduled))
for index, contribution := range scheduled {
got[index] = contribution.identity.resourcePath[1:]
}
if !reflect.DeepEqual(got, []string{"a-base", "b-independent", "z-dependent"}) {
t.Fatalf("import schedule reverse=%t: %v", reverse, got)
}
if !reflect.DeepEqual(
[]string{contributions[0].source.ID, contributions[1].source.ID, contributions[2].source.ID},
[]string{"source-a-base", "source-b-independent", "source-z-dependent"},
) {
t.Fatal("scheduling mutated acquisition source records")
}
inheritanceText := "a-base: {value: ready}\nb-independent: {value: independent}\nz-dependent: {extends: a-base}\n"
if reverse {
inheritanceText = "z-dependent: {extends: a-base}\nb-independent: {value: independent}\na-base: {value: ready}\n"
}
scope := parseCompositionDocument(t, "inheritance", inheritanceText)
inheritance, diagnostics := analyzeInheritanceGraph(scope, Path{})
if len(diagnostics) != 0 {
t.Fatalf("analyze inheritance reverse=%t: %#v", reverse, diagnostics)
}
order, diagnostics := inheritance.schedule()
if len(diagnostics) != 0 {
t.Fatalf("schedule inheritance reverse=%t: %#v", reverse, diagnostics)
}
inheritanceNames := make([]string, len(order))
for index, node := range order {
inheritanceNames[index] = inheritance.nodes[node].name
}
if !reflect.DeepEqual(inheritanceNames, got) {
t.Fatalf("equivalent graph schedules differ reverse=%t: imports=%v inheritance=%v", reverse, got, inheritanceNames)
}
if want == nil {
want = got
} else if !reflect.DeepEqual(got, want) {
t.Fatalf("import graph construction changed schedule: got %v, want %v", got, want)
}
}
}

View File

@@ -0,0 +1,200 @@
package engine
import (
"container/heap"
"sort"
)
// dependencyKey is an immutable tuple used to order dependency-graph nodes.
// Callers provide unique keys and valid edge endpoints.
type dependencyKey []string
// dependencyEdge points from a prerequisite to the node that depends on it.
type dependencyEdge struct {
prerequisite int
dependent int
}
type dependencyReadyHeap struct {
nodes []int
keys []dependencyKey
}
func (h dependencyReadyHeap) Len() int { return len(h.nodes) }
func (h dependencyReadyHeap) Less(i, j int) bool {
return dependencyKeyLess(h.keys[h.nodes[i]], h.keys[h.nodes[j]])
}
func (h dependencyReadyHeap) Swap(i, j int) {
h.nodes[i], h.nodes[j] = h.nodes[j], h.nodes[i]
}
func (h *dependencyReadyHeap) Push(value any) {
h.nodes = append(h.nodes, value.(int))
}
func (h *dependencyReadyHeap) Pop() any {
last := len(h.nodes) - 1
node := h.nodes[last]
h.nodes = h.nodes[:last]
return node
}
// dependencyKeyLess compares tuple components bytewise, with a shorter tuple
// sorting first when all of its components match the corresponding prefix.
func dependencyKeyLess(left, right dependencyKey) bool {
for i := 0; i < len(left) && i < len(right); i++ {
if left[i] != right[i] {
return left[i] < right[i]
}
}
return len(left) < len(right)
}
// scheduleDependencies applies Kahn's algorithm, always selecting the
// smallest currently-ready key. Duplicate edges count only once. If scheduling
// stalls, an explicit depth-first traversal of the residual graph returns an
// actual directed cycle, rotated to its smallest key and closed by repeating
// its first node; blocked non-cycle dependents are excluded.
//
// Nodes are visited by precomputed key rank, which also makes each adjacency
// list deterministic without sorting edges. The work is O(E + V log V), with
// O(V + E) additional memory, apart from key copying and tuple comparisons.
func scheduleDependencies(keys []dependencyKey, edges []dependencyEdge) (order, cycle []int) {
ownedKeys := make([]dependencyKey, len(keys))
for i, key := range keys {
ownedKeys[i] = append(dependencyKey(nil), key...)
}
nodeOrder := make([]int, len(ownedKeys))
for i := range nodeOrder {
nodeOrder[i] = i
}
sort.Slice(nodeOrder, func(i, j int) bool {
return dependencyKeyLess(ownedKeys[nodeOrder[i]], ownedKeys[nodeOrder[j]])
})
incoming := make([][]int, len(ownedKeys))
for _, edge := range edges {
incoming[edge.dependent] = append(incoming[edge.dependent], edge.prerequisite)
}
outgoing := make([][]int, len(ownedKeys))
indegree := make([]int, len(ownedKeys))
for _, dependent := range nodeOrder {
seen := make(map[int]struct{}, len(incoming[dependent]))
for _, prerequisite := range incoming[dependent] {
if _, duplicate := seen[prerequisite]; duplicate {
continue
}
seen[prerequisite] = struct{}{}
indegree[dependent]++
// Dependents are visited by key rank, so each outgoing list is
// already ordered by key.
outgoing[prerequisite] = append(outgoing[prerequisite], dependent)
}
}
ready := dependencyReadyHeap{keys: ownedKeys}
for _, node := range nodeOrder {
if indegree[node] == 0 {
ready.nodes = append(ready.nodes, node)
}
}
heap.Init(&ready)
order = make([]int, 0, len(ownedKeys))
for ready.Len() != 0 {
node := heap.Pop(&ready).(int)
order = append(order, node)
for _, dependent := range outgoing[node] {
indegree[dependent]--
if indegree[dependent] == 0 {
heap.Push(&ready, dependent)
}
}
}
if len(order) == len(ownedKeys) {
return order, nil
}
return nil, dependencyCycle(ownedKeys, nodeOrder, outgoing, indegree)
}
type dependencyDFSFrame struct {
node int
next int
}
// dependencyCycle searches the residual graph iteratively to avoid recursion
// depth depending on input size.
func dependencyCycle(keys []dependencyKey, nodeOrder []int, outgoing [][]int, indegree []int) []int {
const (
unvisited = uint8(0)
active = uint8(1)
finished = uint8(2)
)
state := make([]uint8, len(keys))
position := make([]int, len(keys))
for i := range position {
position[i] = -1
}
path := make([]int, 0, len(keys))
frames := make([]dependencyDFSFrame, 0, len(keys))
for _, root := range nodeOrder {
if indegree[root] == 0 || state[root] != unvisited {
continue
}
state[root] = active
position[root] = len(path)
path = append(path, root)
frames = append(frames, dependencyDFSFrame{node: root})
for len(frames) != 0 {
frame := &frames[len(frames)-1]
if frame.next == len(outgoing[frame.node]) {
state[frame.node] = finished
position[frame.node] = -1
path = path[:len(path)-1]
frames = frames[:len(frames)-1]
continue
}
next := outgoing[frame.node][frame.next]
frame.next++
if indegree[next] == 0 {
continue
}
switch state[next] {
case unvisited:
state[next] = active
position[next] = len(path)
path = append(path, next)
frames = append(frames, dependencyDFSFrame{node: next})
case active:
return canonicalDependencyCycle(keys, path[position[next]:])
}
}
}
// A stalled Kahn traversal always leaves a cycle in the residual graph.
return nil
}
func canonicalDependencyCycle(keys []dependencyKey, path []int) []int {
start := 0
for i := 1; i < len(path); i++ {
if dependencyKeyLess(keys[path[i]], keys[path[start]]) {
start = i
}
}
cycle := make([]int, len(path)+1)
for i := range path {
cycle[i] = path[(start+i)%len(path)]
}
cycle[len(path)] = cycle[0]
return cycle
}

View File

@@ -0,0 +1,215 @@
package engine
import (
"reflect"
"testing"
)
func TestScheduleDependenciesOrdersReadyNodes(t *testing.T) {
tests := []struct {
name string
keys []string
edges [][2]string
want []string
}{
{
name: "independent branches",
keys: []string{"branch-c", "branch-a", "branch-b"},
want: []string{"branch-a", "branch-b", "branch-c"},
},
{
name: "unequal chains",
keys: []string{"chain-d", "chain-c", "chain-b", "chain-a"},
edges: [][2]string{
{"chain-a", "chain-b"},
{"chain-b", "chain-c"},
{"chain-c", "chain-d"},
},
want: []string{"chain-a", "chain-b", "chain-c", "chain-d"},
},
{
name: "multiple dependencies",
keys: []string{"application", "right", "base", "left"},
edges: [][2]string{
{"base", "left"}, {"base", "right"},
{"left", "application"}, {"right", "application"},
},
want: []string{"base", "left", "right", "application"},
},
{
name: "shared dependency at equal depths",
keys: []string{"sink", "right", "root", "left"},
edges: [][2]string{
{"root", "left"}, {"root", "right"},
{"left", "sink"}, {"right", "sink"},
},
want: []string{"root", "left", "right", "sink"},
},
{
name: "shared dependency at unequal depths",
keys: []string{"sink", "deep", "side", "middle", "root"},
edges: [][2]string{
{"root", "middle"}, {"middle", "deep"}, {"root", "side"},
{"deep", "sink"}, {"side", "sink"},
},
want: []string{"root", "middle", "deep", "side", "sink"},
},
{
name: "newly ready small key precedes older large key",
keys: []string{"z-ready", "a-new", "b-ready"},
edges: [][2]string{{"b-ready", "a-new"}},
want: []string{"b-ready", "a-new", "z-ready"},
},
{
name: "duplicate edges count once",
keys: []string{"dependent", "prerequisite"},
edges: [][2]string{
{"prerequisite", "dependent"},
{"prerequisite", "dependent"},
},
want: []string{"prerequisite", "dependent"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
keys, edges, indexes := dependencyGraph(test.keys, test.edges)
order, cycle := scheduleDependencies(keys, edges)
if len(cycle) != 0 {
t.Fatalf("unexpected cycle: %v", dependencyNames(cycle, indexes))
}
if got := dependencyNames(order, indexes); !reflect.DeepEqual(got, test.want) {
t.Fatalf("schedule = %v, want %v", got, test.want)
}
})
}
}
func TestScheduleDependenciesUsesTupleKeys(t *testing.T) {
keys := []dependencyKey{
{"a", "z"},
{"a"},
{"a", "aa"},
{"A"},
}
order, cycle := scheduleDependencies(keys, nil)
if len(cycle) != 0 {
t.Fatalf("unexpected cycle: %v", cycle)
}
want := []int{3, 1, 2, 0}
if !reflect.DeepEqual(order, want) {
t.Fatalf("schedule = %v, want %v", order, want)
}
}
func TestScheduleDependenciesReportsActualCanonicalCycles(t *testing.T) {
tests := []struct {
name string
keys []string
edges [][2]string
want []string
}{
{
name: "self cycle",
keys: []string{"self"},
edges: [][2]string{{"self", "self"}},
want: []string{"self", "self"},
},
{
name: "multi-node cycle excludes blocked dependent",
keys: []string{"blocked", "cycle-b", "cycle-a"},
edges: [][2]string{
{"cycle-a", "cycle-b"},
{"cycle-b", "cycle-a"},
{"cycle-b", "blocked"},
},
want: []string{"cycle-a", "cycle-b", "cycle-a"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
keys, edges, indexes := dependencyGraph(test.keys, test.edges)
order, cycle := scheduleDependencies(keys, edges)
if len(order) != 0 {
t.Fatalf("unexpected schedule: %v", dependencyNames(order, indexes))
}
if got := dependencyNames(cycle, indexes); !reflect.DeepEqual(got, test.want) {
t.Fatalf("cycle = %v, want %v", got, test.want)
}
})
}
}
func TestScheduleDependenciesIsStableAcrossInsertionPermutations(t *testing.T) {
edges := [][2]string{
{"base", "left"}, {"base", "right"},
{"left", "sink"}, {"right", "sink"},
}
permutations := [][]string{
{"sink", "right", "base", "left"},
{"left", "base", "sink", "right"},
{"right", "left", "base", "sink"},
}
for i, permutation := range permutations {
keys, permutedEdges, indexes := dependencyGraph(permutation, edges)
if i%2 != 0 {
reverseDependencyEdges(permutedEdges)
}
order, cycle := scheduleDependencies(keys, permutedEdges)
if len(cycle) != 0 {
t.Fatalf("permutation %d returned cycle %v", i, dependencyNames(cycle, indexes))
}
if got, want := dependencyNames(order, indexes), []string{"base", "left", "right", "sink"}; !reflect.DeepEqual(got, want) {
t.Fatalf("permutation %d schedule = %v, want %v", i, got, want)
}
}
cycleEdges := [][2]string{{"b", "a"}, {"a", "b"}, {"b", "blocked"}}
cyclePermutations := [][]string{{"blocked", "b", "a"}, {"a", "blocked", "b"}, {"b", "a", "blocked"}}
for i, permutation := range cyclePermutations {
keys, permutedEdges, indexes := dependencyGraph(permutation, cycleEdges)
if i%2 != 0 {
reverseDependencyEdges(permutedEdges)
}
order, cycle := scheduleDependencies(keys, permutedEdges)
if len(order) != 0 {
t.Fatalf("cycle permutation %d returned schedule %v", i, dependencyNames(order, indexes))
}
if got, want := dependencyNames(cycle, indexes), []string{"a", "b", "a"}; !reflect.DeepEqual(got, want) {
t.Fatalf("cycle permutation %d returned %v, want %v", i, got, want)
}
}
}
func dependencyGraph(nodeNames []string, namedEdges [][2]string) ([]dependencyKey, []dependencyEdge, map[int]string) {
keys := make([]dependencyKey, len(nodeNames))
indexes := make(map[string]int, len(nodeNames))
for i, name := range nodeNames {
keys[i] = dependencyKey{name}
indexes[name] = i
}
edges := make([]dependencyEdge, len(namedEdges))
for i, edge := range namedEdges {
edges[i] = dependencyEdge{prerequisite: indexes[edge[0]], dependent: indexes[edge[1]]}
}
byIndex := make(map[int]string, len(nodeNames))
for name, index := range indexes {
byIndex[index] = name
}
return keys, edges, byIndex
}
func dependencyNames(nodes []int, names map[int]string) []string {
result := make([]string, len(nodes))
for i, node := range nodes {
result[i] = names[node]
}
return result
}
func reverseDependencyEdges(edges []dependencyEdge) {
for i, j := 0, len(edges)-1; i < j; i, j = i+1, j-1 {
edges[i], edges[j] = edges[j], edges[i]
}
}

View File

@@ -34,7 +34,7 @@ func TestDestinationPlacementCountsAgainstSharedNodeBudget(t *testing.T) {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
t.Parallel() t.Parallel()
accounting := &importAccounting{nodes: limits.nodes - perInput + test.delta} accounting := &importAccounting{nodes: limits.nodes - perInput + test.delta}
composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: empty, destination: destination, source: empty, destination: destination,
}}}, 1, nil, accounting) }}}, 1, nil, accounting)
if test.wantError { if test.wantError {
@@ -62,7 +62,7 @@ func TestMultipleMountedInputsSharePlacementCharges(t *testing.T) {
perInput := countInputNodes(&document.Node{Kind: document.Mapping}) + destinationWrapperNodeCount(Keys("collection", "one")) perInput := countInputNodes(&document.Node{Kind: document.Mapping}) + destinationWrapperNodeCount(Keys("collection", "one"))
charge := len(inputs) * perInput charge := len(inputs) * perInput
accounting := &importAccounting{nodes: limits.nodes - charge} accounting := &importAccounting{nodes: limits.nodes - charge}
composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting) composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting)
if len(diagnostics) != 0 || composition == nil || accounting.nodes != limits.nodes || composition.inputNodes != charge { if len(diagnostics) != 0 || composition == nil || accounting.nodes != limits.nodes || composition.inputNodes != charge {
t.Fatalf("shared placement accounting = %#v, %#v, accounting %#v; want charge %d", composition, diagnostics, accounting, charge) t.Fatalf("shared placement accounting = %#v, %#v, accounting %#v; want charge %d", composition, diagnostics, accounting, charge)
} }
@@ -107,7 +107,7 @@ id: parent
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
accounting := &importAccounting{nodes: limits.nodes - wantNodes + test.delta} accounting := &importAccounting{nodes: limits.nodes - wantNodes + test.delta}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: File(mainPath), imports: true, destination: destination, placementMode: placementModeSequenceItem, source: File(mainPath), imports: true, destination: destination, placementMode: placementModeSequenceItem,
}, { }, {
source: Mapping("other", map[string]any{"id": "other"}), destination: otherDestination, placementMode: placementModeSequenceItem, source: Mapping("other", map[string]any{"id": "other"}), destination: otherDestination, placementMode: placementModeSequenceItem,
@@ -160,7 +160,7 @@ func TestImportedEmptyBodiesChargeEachDestinationWrapper(t *testing.T) {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
t.Parallel() t.Parallel()
accounting := &importAccounting{nodes: limits.nodes - rawNodes - placementNodes + test.delta} accounting := &importAccounting{nodes: limits.nodes - rawNodes - placementNodes + test.delta}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: mainSource, imports: true, destination: destination, source: mainSource, imports: true, destination: destination,
}}}, 1, nil, accounting) }}}, 1, nil, accounting)
if test.wantError { if test.wantError {
@@ -345,11 +345,11 @@ func TestAbsentOptionalImportsDoNotChargeUnusedDeepDestinations(t *testing.T) {
}{ }{
{ {
name: "optional missing path", name: "optional missing path",
body: "configkit: {imports: [{path: missing.yml, optional: true, destination: [unused]}]}\n", body: "configkit: {imports: [{path: missing.yml, destination: [unused]}]}\n",
}, },
{ {
name: "optional unmatched glob", name: "optional unmatched glob",
body: "configkit: {imports: [{glob: empty/*.yml, optional: true, destination: [unused]}]}\n", body: "configkit: {imports: [{glob: empty/*.yml, destination: [unused]}]}\n",
setup: func(t *testing.T, directory string) { setup: func(t *testing.T, directory string) {
if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -387,7 +387,7 @@ func TestNoSourceAndRootPlacementAddNoWrapperCharges(t *testing.T) {
t.Parallel() t.Parallel()
accounting := &importAccounting{} accounting := &importAccounting{}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{ composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{
{source: NoSource(), destination: Keys("absent")}, {source: NoSource(), destination: Keys("absent")},
{source: Mapping("root", map[string]any{})}, {source: Mapping("root", map[string]any{})},
}}, 1, nil, accounting) }}, 1, nil, accounting)
@@ -431,7 +431,13 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) {
overrides: []Override{override}, overrides: []Override{override},
} }
accounting := &importAccounting{nodes: limits.nodes - baseNodes - overrideNodes + 1} accounting := &importAccounting{nodes: limits.nodes - baseNodes - overrideNodes + 1}
_, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting) input, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting)
if len(diagnostics) != 0 {
t.Fatalf("assembly diagnostics = %#v", diagnostics)
}
_, diagnostics = preparePostStructuralOverrides(overridePreparationInput{
root: input.document, inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}, input.sources, request.overrides, nil, limits)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded { if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded {
t.Fatalf("override diagnostics = %#v", diagnostics) t.Fatalf("override diagnostics = %#v", diagnostics)
} }

View File

@@ -451,7 +451,6 @@ func TestLayeredImportOnlyInputMaterializesDestination(t *testing.T) {
mainPath := writeImportFile(t, directory, "main.yml", `configkit: mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports: imports:
- path: optional.yml - path: optional.yml
optional: true
`) `)
loaded, err := LoadLayered( loaded, err := LoadLayered(
Input{Source: File(mainPath), Imports: true, Destination: Keys("collections", "empty")}, Input{Source: File(mainPath), Imports: true, Destination: Keys("collections", "empty")},

View File

@@ -197,8 +197,8 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
t.Parallel() t.Parallel()
definition := Definition[map[string]any]{ definition := Definition[map[string]any]{
Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }}, Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }},
sequenceRegistrations: []sequenceRegistration{{path: Keys("catalog", "entries"), identityField: "id"}}, sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("catalog"), literalSequenceSelector("entries")}, identityFields: []string{"id"}}},
} }
item := func(label string, value map[string]any) Input { item := func(label string, value map[string]any) Input {
return Input{Source: Mapping(label, value), Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem} return Input{Source: Mapping(label, value), Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem}
@@ -209,7 +209,7 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
main := item("main", map[string]any{"id": "main", "schema": map[string]any{"version": "v1"}}) main := item("main", map[string]any{"id": "main", "schema": map[string]any{"version": "v1"}})
t.Run("same assigned main path is protected", func(t *testing.T) { t.Run("same assigned main path is protected", func(t *testing.T) {
_, err := LoadLayered(main, testSecretBinding(), options(item("underlay", map[string]any{"id": "underlay", "schema": nil}))) _, err := LoadLayered(main, testSecretBinding(), options(item("underlay", map[string]any{"id": "main", "schema": nil})))
if got := groupedErrorCode(t, err); got != CodeUnderlayVersion { if got := groupedErrorCode(t, err); got != CodeUnderlayVersion {
t.Fatalf("protected item claim code = %s, want %s", got, CodeUnderlayVersion) t.Fatalf("protected item claim code = %s, want %s", got, CodeUnderlayVersion)
} }
@@ -224,7 +224,7 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
t.Fatalf("unrelated appended peer rejected: %v", err) t.Fatalf("unrelated appended peer rejected: %v", err)
} }
entries := layeredMapValue(t, testLoaded(loaded).Config, "catalog", "entries").([]any) entries := layeredMapValue(t, testLoaded(loaded).Config, "catalog", "entries").([]any)
if len(entries) != 1 || entries[0].(map[string]any)["id"] != "main" { if len(entries) != 3 || entries[0].(map[string]any)["id"] != "first" || entries[1].(map[string]any)["id"] != "peer" || entries[2].(map[string]any)["id"] != "main" {
t.Fatalf("main item projection = %#v", entries) t.Fatalf("main item projection = %#v", entries)
} }
}) })

View File

@@ -9,135 +9,104 @@ import (
const directiveKey = "configkit" const directiveKey = "configkit"
type importDeclaration struct { type rootDirectiveContext uint8
path string
glob string const (
destination Path rootDirectiveContextImportsOnly rootDirectiveContext = iota + 1
placement placementMode rootDirectiveContextSource
placementPath Path rootDirectiveContextOverlay
placementLocation *Location rootDirectiveContextLayered
optional bool )
location Location
type rootDirectiveExtraction struct {
body *document.Node
imports []importDeclarationSpec
sequences []sequenceDeclarationSpec
importsPresent bool
sequencesPresent bool
} }
func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { func splitImportDirectives(root *document.Node) (*document.Node, []importDeclarationSpec, []Diagnostic) {
extracted, diagnostics := extractRootDirectives(root, rootDirectiveContextImportsOnly)
if len(diagnostics) != 0 {
return nil, nil, diagnostics
}
return extracted.body, extracted.imports, nil
}
func extractRootDirectives(root *document.Node, context rootDirectiveContext) (rootDirectiveExtraction, []Diagnostic) {
if root == nil {
return rootDirectiveExtraction{}, nil
}
body := document.Clone(root) body := document.Clone(root)
var declarations []importDeclaration
directiveIndex := -1 directiveIndex := -1
for index, entry := range body.Mapping { for index, entry := range body.Mapping {
if entry.Key != directiveKey { if entry.Key != directiveKey {
continue continue
} }
directiveIndex = index directiveIndex = index
parsed, diagnostics := parseImportDirective(entry.Value, Keys(directiveKey)) extracted, diagnostics := parseRootDirectiveBlock(entry.Value, Keys(directiveKey), context)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
return nil, nil, diagnostics return rootDirectiveExtraction{}, diagnostics
} }
declarations = parsed extracted.body = body
break if directiveIndex >= 0 {
} extracted.body.Mapping = append(extracted.body.Mapping[:directiveIndex], extracted.body.Mapping[directiveIndex+1:]...)
if directiveIndex >= 0 { }
body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...) if diagnostics := nestedDirectiveDiagnostics(extracted.body, Path{}); len(diagnostics) != 0 {
return rootDirectiveExtraction{}, diagnostics
}
return extracted, nil
} }
if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 { if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 {
return nil, nil, diagnostics return rootDirectiveExtraction{}, diagnostics
} }
return body, declarations, nil return rootDirectiveExtraction{body: body}, nil
} }
func parseImportDirective(node *document.Node, directivePath Path) ([]importDeclaration, []Diagnostic) { func parseRootDirectiveBlock(node *document.Node, directivePath Path, context rootDirectiveContext) (rootDirectiveExtraction, []Diagnostic) {
if node.Kind != document.Mapping { if node == nil || node.Kind != document.Mapping {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)} return rootDirectiveExtraction{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)}
} }
var declarations []importDeclaration var extracted rootDirectiveExtraction
var diagnostics []Diagnostic
for _, entry := range node.Mapping { for _, entry := range node.Mapping {
fieldPath := directivePath.Key(entry.Key) fieldPath := directivePath.Key(entry.Key)
if entry.Key != "imports" {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value)}
}
if entry.Value.Kind != document.Sequence {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value)}
}
for index, item := range entry.Value.Sequence {
declaration, diagnostics := parseImportDeclaration(item, fieldPath.Index(index))
if len(diagnostics) != 0 {
return nil, diagnostics
}
declarations = append(declarations, declaration)
}
}
return declarations, nil
}
func parseImportDeclaration(node *document.Node, declarationPath Path) (importDeclaration, []Diagnostic) {
if node.Kind != document.Mapping {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)}
}
declaration := importDeclaration{location: publicLocation(node.Location), placementPath: declarationPath.clone()}
seenTarget := false
for _, entry := range node.Mapping {
fieldPath := declarationPath.Key(entry.Key)
switch entry.Key { switch entry.Key {
case "path", "glob": case "imports":
if seenTarget || entry.Value.Kind != document.String || entry.Value.Text == "" || strings.TrimSpace(entry.Value.Text) != entry.Value.Text { extracted.importsPresent = true
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration requires exactly one nonempty path or glob string", entry.Value)} if context == rootDirectiveContextOverlay {
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeOverlayDirective, fieldPath, "overlays cannot contain imports", entry.Value))
continue
} }
seenTarget = true if context != rootDirectiveContextImportsOnly && context != rootDirectiveContextSource && context != rootDirectiveContextLayered {
declaration.location = publicLocation(entry.Value.Location) diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "directive extraction context is invalid", entry.Value))
if entry.Key == "path" { continue
declaration.path = entry.Value.Text
} else {
declaration.glob = entry.Value.Text
} }
case "optional": parsed, issues := parseImportDeclarations(entry.Value, fieldPath)
if entry.Value.Kind != document.Boolean { extracted.imports = append(extracted.imports, parsed...)
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "optional must be a boolean", entry.Value)} diagnostics = append(diagnostics, issues...)
case "sequences":
extracted.sequencesPresent = true
if context == rootDirectiveContextImportsOnly {
diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "sequence declarations are not enabled in this extraction context", entry.Value))
continue
} }
declaration.optional = entry.Value.Text == "true" if context != rootDirectiveContextSource && context != rootDirectiveContextOverlay && context != rootDirectiveContextLayered {
case "destination": diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "directive extraction context is invalid", entry.Value))
if entry.Value.Kind != document.Sequence { continue
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must be a sequence of strings", entry.Value)}
}
relative := Path{segments: make([]pathSegment, 0, len(entry.Value.Sequence))}
for index, item := range entry.Value.Sequence {
memberPath := fieldPath.Index(index)
if item.Kind != document.String {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be strings", item)}
}
member := Path{segments: []pathSegment{{kind: pathSegmentKey, key: item.Text}}}
if _, err := normalizeDestination(member); err != nil {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be non-reserved mapping keys", item)}
}
relative.segments = append(relative.segments, pathSegment{kind: pathSegmentKey, key: item.Text})
}
normalized, err := normalizeDestination(relative)
if err != nil {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", entry.Value)}
}
declaration.destination = normalized
case "placement":
declaration.placementPath = fieldPath.clone()
placementLocation := publicLocation(entry.Value.Location)
declaration.placementLocation = &placementLocation
if entry.Value.Kind != document.String {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
switch entry.Value.Text {
case "mapping":
declaration.placement = placementModeMapping
case "sequence-item":
declaration.placement = placementModeSequenceItem
default:
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
} }
parsed, issues := parseSequenceDeclarationDirective(entry.Value, fieldPath)
extracted.sequences = append(extracted.sequences, parsed...)
diagnostics = append(diagnostics, issues...)
default: default:
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value))
} }
} }
if !seenTarget { if len(diagnostics) != 0 {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path or glob", node)} return rootDirectiveExtraction{}, canonicalDiagnostics(diagnostics)
} }
return declaration, nil return extracted, nil
} }
func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic {
@@ -197,10 +166,11 @@ func fragmentVersionClaimWithIndex(root *document.Node, rule *VersionRule, sourc
return current return current
} }
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found { if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found {
if index < assigned.start || index >= assigned.end { projected, ok := assigned.inputForOutput(index)
if !ok {
return nil return nil
} }
index -= assigned.start index = projected
} }
if index < 0 || index >= len(current.Sequence) { if index < 0 || index >= len(current.Sequence) {
return nil return nil
@@ -249,24 +219,28 @@ func nodeImportDiagnostic(code Code, path Path, message string, node *document.N
return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location} return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location}
} }
func overlayDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic { func overlayDirectiveDiagnostics(node *document.Node, current Path, allowSequences ...bool) []Diagnostic {
sequencesAllowed := len(allowSequences) != 0 && allowSequences[0]
if node.Kind == document.Mapping { if node.Kind == document.Mapping {
for _, entry := range node.Mapping { for _, entry := range node.Mapping {
childPath := current.Key(entry.Key) childPath := current.Key(entry.Key)
if entry.Key == directiveKey { if entry.Key == directiveKey {
if sequencesAllowed && len(current.segments) == 0 && entry.Value != nil && entry.Value.Kind == document.Mapping && len(entry.Value.Mapping) == 1 && entry.Value.Mapping[0].Key == "sequences" {
continue
}
location := publicLocation(entry.KeyLocation) location := publicLocation(entry.KeyLocation)
return []Diagnostic{{ return []Diagnostic{{
Code: CodeOverlayDirective, Severity: SeverityError, Path: childPath, Code: CodeOverlayDirective, Severity: SeverityError, Path: childPath,
Message: "overlays cannot contain configuration directives", Primary: &location, Message: "overlays may contain only root collection sequence declarations", Primary: &location,
}} }}
} }
if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 { if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath, sequencesAllowed); len(diagnostics) != 0 {
return diagnostics return diagnostics
} }
} }
} else if node.Kind == document.Sequence { } else if node.Kind == document.Sequence {
for index, child := range node.Sequence { for index, child := range node.Sequence {
if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 { if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index), sequencesAllowed); len(diagnostics) != 0 {
return diagnostics return diagnostics
} }
} }
@@ -295,3 +269,72 @@ func disabledDirectiveDiagnostics(node *document.Node, path Path) []Diagnostic {
} }
return diagnostics return diagnostics
} }
func disabledRootImportDiagnostics(node *document.Node) []Diagnostic {
if node == nil || node.Kind != document.Mapping {
return nil
}
var rootDirective *document.Node
var rootLocation document.Location
var foundRootDirective bool
for index := range node.Mapping {
if node.Mapping[index].Key == directiveKey {
rootDirective = node.Mapping[index].Value
rootLocation = node.Mapping[index].KeyLocation
foundRootDirective = true
break
}
}
if foundRootDirective {
if rootDirective == nil || rootDirective.Kind != document.Mapping {
return disabledDirectiveDiagnostic(rootLocation)
}
for _, directive := range rootDirective.Mapping {
if directive.Key == "imports" {
return disabledDirectiveDiagnostic(rootLocation)
}
}
}
for _, entry := range node.Mapping {
if entry.Key == directiveKey {
continue
}
if diagnostic := disabledNestedDirectiveDiagnostics(entry.Value, Path{}.Key(entry.Key)); len(diagnostic) != 0 {
return diagnostic
}
}
return nil
}
func disabledNestedDirectiveDiagnostics(node *document.Node, path Path) []Diagnostic {
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
childPath := path.Key(entry.Key)
if entry.Key == directiveKey {
location := Location{SourceID: entry.KeyLocation.SourceID, Line: entry.KeyLocation.Line, Column: entry.KeyLocation.Column}
return []Diagnostic{{
Code: CodeImportDisabled, Severity: SeverityError, Path: childPath,
Message: "configuration directives require imports to be enabled", Primary: &location,
}}
}
if diagnostics := disabledNestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 {
return diagnostics
}
}
} else if node.Kind == document.Sequence {
for index, child := range node.Sequence {
if diagnostics := disabledNestedDirectiveDiagnostics(child, path.Index(index)); len(diagnostics) != 0 {
return diagnostics
}
}
}
return nil
}
func disabledDirectiveDiagnostic(location document.Location) []Diagnostic {
public := Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column}
return []Diagnostic{{
Code: CodeImportDisabled, Severity: SeverityError, Path: Keys(directiveKey),
Message: "configuration directives require imports to be enabled", Primary: &public,
}}
}

View File

@@ -0,0 +1,353 @@
package engine
import (
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"time"
)
func TestSelectDirectoriesApplyHiddenAndFollowControls(t *testing.T) {
t.Parallel()
root := t.TempDir()
for name, content := range map[string]string{
"direct.yaml": "value: direct\n",
".private.yaml": "value: hidden\n",
".private/nested.yaml": "value: hidden directory\n",
"target/linked.yaml": "value: linked\n",
"target/source.txt": "value: source\n",
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
if err := os.Symlink("target", filepath.Join(root, "visible-alias")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
if err := os.Symlink("target/linked.yaml", filepath.Join(root, "linked-alias.txt")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
if err := os.Symlink("target/source.txt", filepath.Join(root, "selected.yaml")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
if err := os.Symlink("target", filepath.Join(root, ".hidden-alias")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
selection := selectDirectoryWithOptions(t, access, root, ` - dir: .
recursive: true
follow: false
hidden: false
pattern: '*.yaml'
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"direct.yaml", "target/linked.yaml"}; !equalStrings(got, want) {
t.Fatalf("filtered paths = %#v, want %#v", got, want)
}
selection = selectDirectoryWithOptions(t, access, root, ` - dir: visible-alias
follow: false
pattern: '*.yaml'
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"visible-alias/linked.yaml"}; !equalStrings(got, want) {
t.Fatalf("explicit symlink root paths = %#v, want %#v", got, want)
}
selection = selectDirectoryWithOptions(t, access, root, ` - dir: .
recursive: true
follow: true
hidden: false
pattern: '*.yaml'
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"direct.yaml", "selected.yaml", "target/linked.yaml", "visible-alias/linked.yaml"}; !equalStrings(got, want) {
t.Fatalf("followed paths = %#v, want %#v", got, want)
}
selection = selectDirectoryWithOptions(t, access, root, ` - dir: .
recursive: true
follow: true
hidden: true
pattern: '*.yaml'
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{
".hidden-alias/linked.yaml", ".private.yaml", ".private/nested.yaml", "direct.yaml",
"selected.yaml", "target/linked.yaml", "visible-alias/linked.yaml",
}; !equalStrings(got, want) {
t.Fatalf("hidden-enabled paths = %#v, want %#v", got, want)
}
}
func TestSelectDirectoriesRejectEscapingAndCyclicLinks(t *testing.T) {
t.Parallel()
t.Run("escape", func(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "outside.yaml"), []byte("value: outside\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
_, diagnostics := selectDirectoryWithDiagnostics(t, access, root, ` - dir: .
recursive: true
follow: true
pattern: '*.yaml'
`)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportOutsideRoot {
t.Fatalf("escape diagnostics = %#v, want import.outside_root", diagnostics)
}
selection := selectDirectoryWithOptions(t, access, root, ` - dir: .
recursive: true
follow: false
pattern: '*.yaml'
`)
if len(selection.candidates) != 0 {
t.Fatalf("follow:false selected an escaping symlink: %#v", selection.candidates)
}
})
for _, test := range []struct {
name string
}{
{name: "self"},
{name: "ancestor"},
} {
t.Run(test.name, func(t *testing.T) {
root := t.TempDir()
if test.name == "self" {
if err := os.Symlink(".", filepath.Join(root, "loop")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
} else {
if err := os.Mkdir(filepath.Join(root, "nested"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.Symlink("..", filepath.Join(root, "nested", "up")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
_, diagnostics := selectDirectoryWithDiagnostics(t, access, root, ` - dir: .
recursive: true
follow: true
`)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportCycle {
t.Fatalf("cycle diagnostics = %#v, want import.cycle", diagnostics)
}
if diagnostics[0].Message != "directory traversal cycle detected" {
t.Fatalf("cycle diagnostic exposes unexpected detail: %q", diagnostics[0].Message)
}
})
}
}
func TestSelectDirectoriesRetainSeparateNoncyclicAliases(t *testing.T) {
t.Parallel()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "target"), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "target", "item.yaml"), []byte("value: item\n"), 0o600); err != nil {
t.Fatal(err)
}
for _, alias := range []string{"first", "second"} {
if err := os.Symlink("target", filepath.Join(root, alias)); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
selection := selectDirectoryWithOptions(t, access, root, ` - dir: .
recursive: true
follow: true
pattern: '*.yaml'
`)
want := []string{"first/item.yaml", "second/item.yaml", "target/item.yaml"}
if got := selectedPathsRelativeTo(selection, root); !equalStrings(got, want) {
t.Fatalf("alias candidates = %#v, want %#v", got, want)
}
}
func TestSelectDirectoriesUseNativeReadLinkCapability(t *testing.T) {
t.Parallel()
filesystem := &linkAwareMapFS{
files: fstest.MapFS{
"real": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"real/item.yaml": &fstest.MapFile{Data: []byte("value: item\n")},
"alias": &fstest.MapFile{Mode: fs.ModeSymlink, Data: []byte("real")},
".hidden": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
".hidden/secret.yaml": &fstest.MapFile{Data: []byte("value: secret\n")},
},
links: map[string]string{"alias": "real"},
}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
selection := selectDirectoryWithOptions(t, access, ".", ` - dir: .
recursive: true
follow: true
hidden: false
pattern: '*.yaml'
`)
if got, want := selectedPaths(selection), []string{"alias/item.yaml", "real/item.yaml"}; !equalStrings(got, want) {
t.Fatalf("native link candidates = %#v, want %#v", got, want)
}
cyclic := &linkAwareMapFS{
files: fstest.MapFS{
"loop": &fstest.MapFile{Mode: fs.ModeSymlink, Data: []byte(".")},
},
links: map[string]string{"loop": "."},
}
_, diagnostics := selectDirectoryWithDiagnostics(t, newNativeAccess(cyclic, ".", &namespaceToken{}), ".", ` - dir: .
recursive: true
follow: true
`)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportCycle {
t.Fatalf("native loop diagnostics = %#v, want import.cycle", diagnostics)
}
plain := newNativeAccess(fstest.MapFS{"plain.yaml": &fstest.MapFile{Data: []byte("value: plain\n")}}, ".", &namespaceToken{})
selection = selectDirectoryWithOptions(t, plain, ".", " - dir: .\n")
if got, want := selectedPaths(selection), []string{"plain.yaml"}; !equalStrings(got, want) {
t.Fatalf("plain native filesystem selection = %#v, want %#v", got, want)
}
}
func selectDirectoryWithOptions(t *testing.T, access *filesystemAccess, base, declaration string) importSelection {
t.Helper()
selection, diagnostics := selectDirectoryWithDiagnostics(t, access, base, declaration)
if len(diagnostics) != 0 {
t.Fatalf("directory selection diagnostics = %#v", diagnostics)
}
return selection
}
func selectDirectoryWithDiagnostics(t *testing.T, access *filesystemAccess, base, declaration string) (importSelection, []Diagnostic) {
t.Helper()
specs, diagnostics := parseImportSpecsYAML(t, declaration)
if len(diagnostics) != 0 {
t.Fatalf("parse diagnostics = %#v", diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
return selectDirectoryImportCandidates(access, base, specs[0], policy, defaultImportLimits(), &importAccounting{})
}
func selectedPaths(selection importSelection) []string {
paths := make([]string, 0, len(selection.candidates))
for _, candidate := range selection.candidates {
paths = append(paths, candidate.path)
}
return paths
}
func selectedPathsRelativeTo(selection importSelection, root string) []string {
paths := make([]string, 0, len(selection.candidates))
for _, candidate := range selection.candidates {
relative, err := filepath.Rel(root, candidate.path)
if err != nil {
paths = append(paths, candidate.path)
continue
}
paths = append(paths, filepath.ToSlash(relative))
}
return paths
}
func equalStrings(left, right []string) bool {
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}
type linkAwareMapFS struct {
files fstest.MapFS
links map[string]string
}
func (f *linkAwareMapFS) Open(name string) (fs.File, error) {
resolved, err := f.resolve(name)
if err != nil {
return nil, err
}
return f.files.Open(resolved)
}
func (f *linkAwareMapFS) ReadLink(name string) (string, error) {
target, ok := f.links[name]
if !ok {
return "", &fs.PathError{Op: "readlink", Path: name, Err: fs.ErrInvalid}
}
return target, nil
}
func (f *linkAwareMapFS) Lstat(name string) (fs.FileInfo, error) {
if _, ok := f.links[name]; ok {
return linkMapFileInfo{name: path.Base(name)}, nil
}
return fs.Stat(f.files, name)
}
func (f *linkAwareMapFS) resolve(name string) (string, error) {
if !fs.ValidPath(name) {
return "", fs.ErrInvalid
}
resolved := ""
seen := make(map[string]struct{})
for _, segment := range strings.Split(name, "/") {
candidate := path.Join(resolved, segment)
if target, ok := f.links[candidate]; ok {
if _, exists := seen[candidate]; exists {
return "", fs.ErrInvalid
}
seen[candidate] = struct{}{}
candidate = path.Clean(path.Join(path.Dir(candidate), target))
}
resolved = candidate
}
return resolved, nil
}
type linkMapFileInfo struct{ name string }
func (i linkMapFileInfo) Name() string { return i.name }
func (linkMapFileInfo) Size() int64 { return 0 }
func (linkMapFileInfo) Mode() fs.FileMode { return fs.ModeSymlink | 0o777 }
func (linkMapFileInfo) ModTime() time.Time { return time.Time{} }
func (linkMapFileInfo) IsDir() bool { return false }
func (linkMapFileInfo) Sys() any { return nil }

View File

@@ -0,0 +1,371 @@
package engine
import (
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
)
func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) {
t.Parallel()
filesystem := fstest.MapFS{
"one": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"one/a.yaml": &fstest.MapFile{Data: []byte("a: 1\n")},
"one/README": &fstest.MapFile{Data: []byte("readme\n")},
"one/.hidden": &fstest.MapFile{Data: []byte("hidden\n")},
"one/draft.draft.yaml": &fstest.MapFile{Data: []byte("draft\n")},
"one/nested/deep.yaml": &fstest.MapFile{Data: []byte("deep: 1\n")},
"two": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"two/a.yaml": &fstest.MapFile{Data: []byte("a: 2\n")},
"three": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"three/extensionless": &fstest.MapFile{Data: []byte("plain\n")},
}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, `
- dir: [one, one, missing, two]
pattern: ['*.yaml', 'a.*', 'README']
exclude: '*.draft.yaml'
required: true
`)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
accounting := &importAccounting{}
selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
if selection.outcome != importSelectionHasCandidates || len(selection.candidates) != 3 {
t.Fatalf("selection = %#v, want three unioned matches", selection)
}
if selection.candidates[0].path != "one/README" || selection.candidates[1].path != "one/a.yaml" || selection.candidates[2].path != "two/a.yaml" {
t.Fatalf("selected paths = %#v", selection.candidates)
}
if accounting.entries != 6 {
t.Fatalf("entry visits = %d, want 6 entries across distinct roots", accounting.entries)
}
}
func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(t *testing.T) {
t.Parallel()
root := t.TempDir()
for name, value := range map[string]string{
"a.yaml": "a: 1\n",
".private": "private\n",
"README": "readme\n",
"nested/deep": "deep\n",
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte(value), 0o600); err != nil {
t.Fatal(err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
if len(selection.candidates) != 3 {
t.Fatalf("selected files = %#v, want yaml, hidden, and extensionless files", selection.candidates)
}
for _, candidate := range selection.candidates {
if filepath.Base(candidate.path) == "deep" {
t.Fatalf("shallow search descended into a child directory: %#v", selection.candidates)
}
}
}
func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) {
t.Parallel()
filesystem := fstest.MapFS{
"available": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"available/match.yaml": &fstest.MapFile{Data: []byte("match: true\n")},
"available/excluded.yaml": &fstest.MapFile{Data: []byte("excluded: true\n")},
}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
for _, test := range []struct {
name string
yaml string
wantError bool
wantCount int
}{
{
name: "missing root alongside a match",
yaml: " - dir: [missing, available]\n pattern: '*.yaml'\n exclude: excluded.yaml\n required: true\n",
wantCount: 1,
},
{
name: "all matches excluded",
yaml: " - dir: available\n pattern: '*.yaml'\n exclude: '*.yaml'\n required: true\n",
wantError: true,
},
{
name: "explicit empty patterns select nothing",
yaml: " - dir: available\n pattern: []\n",
wantCount: 0,
},
{
name: "empty search roots",
yaml: " - dir: []\n required: true\n",
wantError: true,
},
} {
t.Run(test.name, func(t *testing.T) {
specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if test.wantError {
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics)
}
return
}
if len(diagnostics) != 0 || len(selection.candidates) != test.wantCount {
t.Fatalf("selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if test.wantCount == 1 && selection.candidates[0].path != "available/match.yaml" {
t.Fatalf("selected path = %q, want available/match.yaml", selection.candidates[0].path)
}
})
}
}
func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t *testing.T) {
t.Parallel()
filesystem := fstest.MapFS{
"file.yaml": &fstest.MapFile{Data: []byte("not a directory\n")},
}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, " - dir: file.yaml\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
t.Fatalf("invalid-root diagnostics = %#v", diagnostics)
}
denied := &statOnlyImportFilesystem{statErrors: map[string]error{"denied": fs.ErrPermission}}
access = newNativeAccess(denied, ".", &namespaceToken{})
specs, diagnostics = parseImportSpecsYAML(t, " - dir: denied\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics = resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics)
}
}
func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) {
t.Parallel()
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "welcome.yaml"), []byte("welcome: true\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("welcome.yaml", filepath.Join(root, "welcome.draft.yaml")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n pattern: '*.yaml'\n exclude: '*.draft.yaml'\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" {
t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics)
}
}
func TestSelectDirectoriesApplyRecursiveDepthToFileLevels(t *testing.T) {
t.Parallel()
root := t.TempDir()
for name := range map[string]struct{}{
"direct.yaml": {},
"child/child.yaml": {},
"child/grand/grand.yaml": {},
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
t.Fatal(err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
for _, test := range []struct {
name string
yaml string
defaults importDefaults
wantPaths []string
}{
{
name: "depth one includes direct files",
yaml: " - dir: .\n recursive: true\n depth: 1\n pattern: '*.yaml'\n",
wantPaths: []string{"direct.yaml"},
},
{
name: "depth two includes one descendant level",
yaml: " - dir: .\n recursive: true\n depth: 2\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "direct.yaml"},
},
{
name: "zero depth is unlimited",
yaml: " - dir: .\n recursive: true\n depth: 0\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
},
{
name: "recursion stays disabled by default",
yaml: " - dir: .\n pattern: '*.yaml'\n",
wantPaths: []string{"direct.yaml"},
},
{
name: "explicit false overrides application recursion",
yaml: " - dir: .\n recursive: false\n pattern: '*.yaml'\n",
defaults: importDefaults{recursive: boolPointer(true)},
wantPaths: []string{"direct.yaml"},
},
{
name: "application default enables recursion for depth",
yaml: " - dir: .\n depth: 2\n pattern: '*.yaml'\n",
defaults: importDefaults{recursive: boolPointer(true)},
wantPaths: []string{"child.yaml", "direct.yaml"},
},
{
name: "large depth remains a valid unlimited traversal bound",
yaml: " - dir: .\n recursive: true\n depth: 999999999999999999999999999999\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
},
{
name: "filename filter does not prune directories",
yaml: " - dir: .\n recursive: true\n depth: 3\n pattern: grand.yaml\n",
wantPaths: []string{"grand.yaml"},
},
} {
t.Run(test.name, func(t *testing.T) {
specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], test.defaults)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
var got []string
for _, candidate := range selection.candidates {
got = append(got, filepath.Base(candidate.path))
}
if strings.Join(got, ",") != strings.Join(test.wantPaths, ",") {
t.Fatalf("selected basenames = %#v, want %#v", got, test.wantPaths)
}
})
}
}
func TestSelectDirectoriesRetainsEmptySearchAndDirectoryVisitLimits(t *testing.T) {
t.Parallel()
root := t.TempDir()
for _, name := range []string{"direct.yaml", "child/child.yaml", "child/grand/grand.yaml"} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
t.Fatal(err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n recursive: true\n pattern: []\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
accounting := &importAccounting{}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), accounting)
if len(diagnostics) != 0 || selection.outcome != importSelectionNone || len(selection.candidates) != 0 {
t.Fatalf("empty recursive selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if accounting.entries != 5 {
t.Fatalf("empty search visited %d entries, want all 5 within the recursive search", accounting.entries)
}
limited := defaultImportLimits()
limited.entries = 2
_, diagnostics = selectDirectoryImportCandidates(access, root, specs[0], policy, limited, &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("directory limit diagnostics = %#v", diagnostics)
}
}
func boolPointer(value bool) *bool { return &value }

View File

@@ -0,0 +1,32 @@
package engine
import (
"errors"
"io/fs"
)
var errImportDirectoryEntryLimit = errors.New("import directory-entry visit limit reached")
// visitBoundedImportDirectory shares entry accounting across directory and glob searches.
func visitBoundedImportDirectory(access *filesystemAccess, directory string, limit int, accounting *importAccounting, visit func(fs.DirEntry) error) error {
if accounting == nil {
accounting = &importAccounting{}
}
var visitError error
err := access.readDirectory(directory, func(entry fs.DirEntry) bool {
accounting.entries++
if accounting.entries > limit {
visitError = errImportDirectoryEntryLimit
return false
}
if err := visit(entry); err != nil {
visitError = err
return false
}
return true
})
if visitError != nil {
return visitError
}
return err
}

View File

@@ -76,20 +76,55 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) {
// Intermediate input replacements do not permanently suppress defaults: // Intermediate input replacements do not permanently suppress defaults:
// decoding applies them to the final input tree. Show that reconciliation // decoding applies them to the final input tree. Show that reconciliation
// explicitly, including canonical scalar conversions, before domain changes. // explicitly before typed CLI contributions and domain resolution.
decodedInput := r.preCLI
decodedOrigins := r.preCLIOrigins
replayed, _ := documentAtPath(working, path) replayed, _ := documentAtPath(working, path)
decoded, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup) decoded, _ := documentAtPathWithLookup(decodedInput, path, r.immutableLookup)
if !configurationValuesEqual(replayed, decoded) { if !configurationValuesEqual(replayed, decoded) {
origin := cloneOrigin(decodedOrigins[path.String()])
origin.Operation = ChangeDecoding
explanation.History = append(explanation.History, ValueChange{
Before: valueSnapshot(working, path, r.secrets),
After: valueSnapshotWithLookup(decodedInput, path, r.secrets, r.immutableLookup),
Origin: origin,
Reasons: []string{"applied retained defaults and canonical typed decoding"},
})
}
working = document.Clone(decodedInput)
for _, application := range r.postDecodeApps {
operationPath, relevant := applicationOperationPath(application, path, r.immutableLookup)
var before ValueSnapshot
if relevant {
before = valueSnapshot(working, path, r.secrets)
}
applyHistoryApplication(&working, application, replayIndex)
if !relevant {
continue
}
after := valueSnapshot(working, path, r.secrets)
if !before.Present && !after.Present {
continue
}
explanation.History = append(explanation.History, ValueChange{
Before: before,
After: after,
Origin: historyApplicationOrigin(application, operationPath, r.immutableLookup),
})
}
replayed, _ = documentAtPath(working, path)
effectiveInput, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup)
if !configurationValuesEqual(replayed, effectiveInput) {
origin := cloneOrigin(r.inputOrigins[path.String()]) origin := cloneOrigin(r.inputOrigins[path.String()])
origin.Operation = ChangeDecoding origin.Operation = ChangeDecoding
explanation.History = append(explanation.History, ValueChange{ explanation.History = append(explanation.History, ValueChange{
Before: valueSnapshot(working, path, r.secrets), Before: valueSnapshot(working, path, r.secrets),
After: valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup), After: valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup),
Origin: origin, Origin: origin,
Reasons: []string{"applied retained defaults and canonical typed decoding"}, Reasons: []string{"applied the typed CLI assignment"},
}) })
} }
working = document.Clone(r.input) working = document.Clone(r.input)
for _, snapshot := range r.resolutionReplay { for _, snapshot := range r.resolutionReplay {
relevant := pathsIntersect(snapshot.change.Path, path) relevant := pathsIntersect(snapshot.change.Path, path)

View File

@@ -351,6 +351,10 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) {
opts.Overlay = "p" opts.Overlay = "p"
} }
result, err := testLoad(File(main), opts) result, err := testLoad(File(main), opts)
if cli && base == "replaced-text" {
requireSecretLoadCode(t, err, CodeDecodeType)
return
}
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -360,6 +364,18 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) {
t.Fatal("missing explanation") t.Fatal("missing explanation")
} }
last := explanation.History[len(explanation.History)-1] last := explanation.History[len(explanation.History)-1]
if cli && path.String() == "/m" {
if last.After != explanation.Effective || last.Origin.Layer != LayerCLI || last.Origin.Operation != "" {
t.Fatalf("CLI assignment was not the final history event: %#v", explanation)
}
continue
}
if cli && path.String() == "/m/a" {
if explanation.Effective.Present {
t.Fatalf("CLI map override retained descendants discarded by source null: %#v", explanation)
}
continue
}
if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding { if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding {
t.Fatalf("history does not reconcile final value: %#v", explanation) t.Fatalf("history does not reconcile final value: %#v", explanation)
} }

View File

@@ -0,0 +1,247 @@
package engine
import (
"errors"
"io/fs"
"path"
"path/filepath"
"strings"
)
var (
errImportDirectoryLinkCycle = errors.New("directory link cycle")
errImportPathOutsideRoot = errors.New("resolved import path is outside the configuration root")
)
// importEntryTarget describes a discovered entry without changing its lexical
// candidate spelling. identity is resolved only to detect active traversal loops.
type importEntryTarget struct {
symlink bool
dir bool
identity string
}
func (a *filesystemAccess) importEntryTarget(name string, entry fs.DirEntry, follow bool) (importEntryTarget, error) {
if a.native == nil {
relative, err := a.relative(name)
if err != nil {
return importEntryTarget{}, err
}
// Root.Lstat cannot follow absolute links in intermediate components,
// even when contained. Resolve parents first, leaving the final entry
// unexpanded so its discovered-link follow policy still applies.
if parent := filepath.Dir(relative); parent != "." {
resolvedParent, err := resolveHostImportPath(a, parent)
if err != nil {
return importEntryTarget{}, err
}
relative = filepath.Join(filepath.FromSlash(resolvedParent), filepath.Base(relative))
}
info, err := a.root.Lstat(relative)
if err != nil {
return importEntryTarget{}, err
}
symlink := info.Mode()&fs.ModeSymlink != 0
if symlink && !follow {
return importEntryTarget{symlink: true}, nil
}
if !symlink && !info.IsDir() {
return importEntryTarget{}, nil
}
resolved, err := resolveHostImportPath(a, relative)
if err != nil {
return importEntryTarget{}, err
}
resolvedPath := filepath.Join(a.rootPath, filepath.FromSlash(resolved))
info, err = a.root.Stat(filepath.FromSlash(resolved))
if err != nil {
return importEntryTarget{}, err
}
resolvedPath = filepath.Clean(resolvedPath)
return importEntryTarget{symlink: symlink, dir: info.IsDir(), identity: resolvedPath}, nil
}
linkFS, linksKnown := a.native.(fs.ReadLinkFS)
if linksKnown {
info, err := linkFS.Lstat(name)
if err != nil {
return importEntryTarget{}, err
}
symlink := info.Mode()&fs.ModeSymlink != 0
if symlink && !follow {
return importEntryTarget{symlink: true}, nil
}
if !symlink && !info.IsDir() {
return importEntryTarget{}, nil
}
resolved, err := resolveNativeImportPath(a, linkFS, name)
if err != nil {
return importEntryTarget{}, err
}
if !a.contains(resolved) {
return importEntryTarget{}, errImportPathOutsideRoot
}
info, err = fs.Stat(a.native, resolved)
if err != nil {
return importEntryTarget{}, err
}
return importEntryTarget{symlink: symlink, dir: info.IsDir(), identity: resolved}, nil
}
symlink := entry != nil && entry.Type()&fs.ModeSymlink != 0
if symlink && !follow {
return importEntryTarget{symlink: true}, nil
}
if entry != nil && !symlink {
mode := entry.Type()
if mode == 0 {
info, err := entry.Info()
if err != nil {
return importEntryTarget{}, err
}
mode = info.Mode()
symlink = mode&fs.ModeSymlink != 0
if symlink && !follow {
return importEntryTarget{symlink: true}, nil
}
}
if !symlink && !mode.IsDir() {
return importEntryTarget{}, nil
}
if !symlink && mode.IsDir() {
return importEntryTarget{dir: true, identity: name}, nil
}
}
info, err := fs.Stat(a.native, name)
if err != nil {
return importEntryTarget{}, err
}
return importEntryTarget{symlink: symlink, dir: info.IsDir(), identity: name}, nil
}
// A link stays active until its target has resolved, not until the entire
// original pathname has resolved. Remaining segments may reuse a finished link.
type importLinkFrame struct {
pending []string
link string
}
func resolveHostImportPath(access *filesystemAccess, name string) (string, error) {
frames := []importLinkFrame{{pending: strings.Split(filepath.ToSlash(name), "/")}}
resolved := ""
activeLinks := make(map[string]struct{})
for len(frames) > 0 {
frame := &frames[len(frames)-1]
if len(frame.pending) == 0 {
delete(activeLinks, frame.link)
frames = frames[:len(frames)-1]
continue
}
segment := frame.pending[0]
frame.pending = frame.pending[1:]
candidate := path.Join(resolved, segment)
if candidate == "" {
candidate = "."
}
info, err := access.root.Lstat(filepath.FromSlash(candidate))
if err != nil {
return "", err
}
if info.Mode()&fs.ModeSymlink == 0 {
resolved = candidate
continue
}
if _, exists := activeLinks[candidate]; exists {
return "", errImportDirectoryLinkCycle
}
activeLinks[candidate] = struct{}{}
target, err := access.root.Readlink(filepath.FromSlash(candidate))
if err != nil {
return "", err
}
var targetPath string
if filepath.IsAbs(target) {
relative, err := filepath.Rel(access.rootPath, filepath.Clean(target))
if err != nil {
return "", errImportPathOutsideRoot
}
targetPath = path.Clean(filepath.ToSlash(relative))
} else {
targetPath = path.Clean(path.Join(path.Dir(candidate), filepath.ToSlash(target)))
}
if !withinPortableRoot(targetPath) {
return "", errImportPathOutsideRoot
}
frames = append(frames, importLinkFrame{pending: strings.Split(targetPath, "/"), link: candidate})
resolved = ""
}
if resolved == "" {
resolved = "."
}
return resolved, nil
}
func withinPortableRoot(name string) bool {
return name == "." || name != ".." && !strings.HasPrefix(name, "../") && !strings.HasPrefix(name, "/")
}
func resolveNativeImportPath(access *filesystemAccess, filesystem fs.ReadLinkFS, name string) (string, error) {
if !fs.ValidPath(name) || !access.contains(name) {
return "", fs.ErrInvalid
}
frames := []importLinkFrame{{pending: strings.Split(name, "/")}}
resolved := ""
activeLinks := make(map[string]struct{})
for len(frames) > 0 {
frame := &frames[len(frames)-1]
if len(frame.pending) == 0 {
delete(activeLinks, frame.link)
frames = frames[:len(frames)-1]
continue
}
segment := frame.pending[0]
frame.pending = frame.pending[1:]
candidate := path.Join(resolved, segment)
if candidate == "" {
candidate = "."
}
// Ancestors of a multi-component root are resolution prefixes, not
// selected targets. Complete paths and link targets remain contained.
if !access.contains(candidate) && !strings.HasPrefix(access.rootPath, candidate+"/") {
return "", errImportPathOutsideRoot
}
info, err := filesystem.Lstat(candidate)
if err != nil {
return "", err
}
if info.Mode()&fs.ModeSymlink == 0 {
resolved = candidate
continue
}
if _, exists := activeLinks[candidate]; exists {
return "", errImportDirectoryLinkCycle
}
activeLinks[candidate] = struct{}{}
target, err := filesystem.ReadLink(candidate)
if err != nil {
return "", err
}
if target == "" || strings.HasPrefix(target, "/") {
return "", errImportPathOutsideRoot
}
targetPath := path.Clean(path.Join(path.Dir(candidate), target))
if !fs.ValidPath(targetPath) || !access.contains(targetPath) {
return "", errImportPathOutsideRoot
}
frames = append(frames, importLinkFrame{pending: strings.Split(targetPath, "/"), link: candidate})
resolved = ""
}
if resolved == "" {
resolved = "."
}
return resolved, nil
}
func isImportLinkCycle(err error) bool {
return errors.Is(err, errImportDirectoryLinkCycle)
}

View File

@@ -0,0 +1,47 @@
package engine
import (
"errors"
"io/fs"
"os"
"path/filepath"
"testing"
)
func TestImportLinkResolutionTracksActiveExpansions(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "parts"), 0o700); err != nil {
t.Fatal(err)
}
for name, target := range map[string]string{"parts/up": "..", "self": "self", "first": "second", "second": "first"} {
if err := os.Symlink(target, filepath.Join(root, filepath.FromSlash(name))); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
}
host, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer host.close()
filesystem := os.DirFS(root)
links, ok := filesystem.(fs.ReadLinkFS)
if !ok {
t.Fatal("native directory filesystem does not expose link inspection")
}
native := newNativeAccess(filesystem, ".", &namespaceToken{})
for name, resolve := range map[string]func(string) (string, error){
"host": func(name string) (string, error) { return resolveHostImportPath(host, name) },
"native": func(name string) (string, error) { return resolveNativeImportPath(native, links, name) },
} {
t.Run(name, func(t *testing.T) {
if got, err := resolve("parts/up/parts/up/parts"); err != nil || got != "parts" {
t.Errorf("finite link reuse = %q, %v", got, err)
}
for _, cycle := range []string{"self", "first"} {
if _, err := resolve(cycle); !errors.Is(err, errImportDirectoryLinkCycle) {
t.Errorf("%s cycle = %v", cycle, err)
}
}
})
}
}

View File

@@ -0,0 +1,51 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func validateFinalSequenceIdentities(root *document.Node, policy sequencePolicy) []Diagnostic {
if root == nil || policy.selectorTrie == nil {
return nil
}
var diagnostics []Diagnostic
var visit func(*document.Node, Path, []*sequenceSelectorTrie)
visit = func(node *document.Node, path Path, states []*sequenceSelectorTrie) {
if node == nil {
return
}
if node.Kind == document.Sequence {
var fields []string
for _, state := range states {
if state.terminal {
fields = state.fields
break
}
}
if fields != nil {
_, issues := validateNormalizedSequenceIdentityFrom(node, path, fields, sequenceIdentityDiagnosticCodes{
invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict,
}, 0, sequencePayloadRecord)
diagnostics = append(diagnostics, issues...)
}
return
}
if node.Kind != document.Mapping {
return
}
for _, entry := range node.Mapping {
next := make([]*sequenceSelectorTrie, 0, len(states)*2)
for _, state := range states {
if literal := state.literals[entry.Key]; literal != nil {
next = append(next, literal)
}
if state.wildcard != nil {
next = append(next, state.wildcard)
}
}
if len(next) != 0 {
visit(entry.Value, path.Key(entry.Key), next)
}
}
}
visit(root, Path{}, []*sequenceSelectorTrie{policy.selectorTrie})
return canonicalDiagnostics(diagnostics)
}

View File

@@ -5,130 +5,223 @@ import (
"io/fs" "io/fs"
"path" "path"
"slices" "slices"
"sort"
"strings" "strings"
) )
const globReadBatch = 128 const globReadBatch = 128
type globCandidate struct { // selectGlobImportCandidates expands path components while retaining the
absolute string // lexical spelling used for filtering and later source admission.
relative string func selectGlobImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
} site := spec.fieldSites["glob"]
segments := spec.globSegments
func (s *importTraversal) expandImportGlob(basePath string, declaration importDeclaration) ([]string, []Diagnostic) { if len(segments) == 0 {
location := declaration.location var valid bool
segments, valid := validGlobPattern(declaration.glob) segments, valid = validGlobPattern(spec.glob)
if !valid { if !valid {
return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid or uses recursive ** syntax", &location)} return importSelection{}, []Diagnostic{*importSelectionDiagnostic(site, CodeImportInvalid, "import glob pattern is invalid")}
}
} }
if accounting == nil {
candidates := []globCandidate{{absolute: basePath}} accounting = &importAccounting{}
}
rootPath := normalizedImportCandidatePath(access, basePath)
candidates := []selectedImportCandidate{{path: rootPath, accessPath: basePath, site: site}}
for index, segment := range segments { for index, segment := range segments {
last := index == len(segments)-1 last := index == len(segments)-1
literal, patterned := globLiteralSegment(segment) literal, patterned := globLiteralSegment(segment)
var next []globCandidate // nextByPath is the set union of candidates reached from each matching
for _, candidate := range candidates { // parent at this segment; final exclusions below are set difference.
var names []string nextByPath := make(map[string]selectedImportCandidate)
if patterned { for _, parent := range candidates {
matched, diagnostics := s.readGlobMatches(candidate.absolute, segment, location) if !patterned {
childPath := access.join(parent.path, literal)
childAccessPath := access.join(parent.accessPath, literal)
candidate, missing, diagnostics := explicitGlobCandidate(access, basePath, childPath, childAccessPath, site, last)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
return nil, diagnostics return importSelection{}, diagnostics
} }
names = matched if !missing {
} else { nextByPath[candidate.path] = candidate
names = []string{literal} }
continue
} }
for _, name := range names {
absolute := s.access.join(candidate.absolute, name) entries, missing, diagnostics := readGlobEntries(access, parent.accessPath, site.location, limits, accounting)
relative := path.Join(candidate.relative, name) if len(diagnostics) != 0 {
if !s.access.contains(absolute) { return importSelection{}, diagnostics
return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal is outside the configuration root", &location)} }
} if missing {
canonical, err := s.access.canonical(absolute) continue
}
for _, entry := range entries {
matched, err := path.Match(segment, entry.Name())
if err != nil { if err != nil {
if errors.Is(err, fs.ErrNotExist) && !patterned { return importSelection{}, []Diagnostic{*importSelectionDiagnostic(site, CodeImportInvalid, "import glob pattern is invalid")}
continue
}
return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", location)}
} }
if !s.access.contains(canonical) { if !matched || !policy.hidden && strings.HasPrefix(entry.Name(), ".") {
return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob traversal resolves outside the configuration root", &location)}
}
if last {
next = append(next, globCandidate{absolute: canonical, relative: relative})
if s.accounting.sources+len(next) > s.limits.sources {
return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob matches exceed the remaining source-count limit", &location)}
}
continue continue
} }
info, err := s.access.stat(canonical) accessPath := access.join(parent.accessPath, entry.Name())
lexicalPath := access.join(parent.path, entry.Name())
if !access.contains(accessPath) || !access.contains(lexicalPath) {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(site, CodeImportOutsideRoot, "glob traversal is outside the configuration root")}
}
target, err := access.importEntryTarget(accessPath, entry, policy.follow)
if err != nil { if err != nil {
return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob traversal target could not be inspected", location)} return importSelection{}, []Diagnostic{globCandidateDiagnostic(err, site.location, "glob match could not be inspected")}
} }
if info.IsDir() { if target.symlink && !policy.follow {
next = append(next, globCandidate{absolute: canonical, relative: relative}) continue
} }
if !last && !target.dir {
continue
}
physicalPath := target.identity
// Native source identity and relative imports use the namespace
// spelling; resolved link identities belong only to discovery.
if access.native != nil {
physicalPath = accessPath
}
if physicalPath == "" {
physicalPath, err = access.canonical(accessPath)
if err != nil {
return importSelection{}, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", site.location)}
}
}
candidate := selectedImportCandidate{
path: normalizedImportCandidatePath(access, lexicalPath), accessPath: physicalPath,
label: relativeImportLabel(access, basePath, lexicalPath), site: site,
}
nextByPath[candidate.path] = candidate
} }
} }
candidates = next candidates = candidatesFromMap(nextByPath)
if len(candidates) == 0 { if len(candidates) == 0 {
return nil, nil break
} }
} }
matches := make([]string, len(candidates)) selected := make([]selectedImportCandidate, 0, len(candidates))
for index, candidate := range candidates { for _, candidate := range candidates {
matches[index] = candidate.relative missing, _, diagnostic := validateNamedImportTarget(access, candidate, false, false)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if missing || excludedBasename(path.Base(candidate.path), policy.exclude) {
continue
}
selected = append(selected, candidate)
} }
slices.Sort(matches) if policy.required && len(selected) == 0 {
return matches, nil return importSelection{}, []Diagnostic{*importSelectionDiagnostic(site, CodeImportMissing, "required import glob selected no files")}
}
// Source allowance is charged by source admission, after this filtered set
// reaches the traversal; that preserves repeated-source diagnostics first.
selection := importSelection{candidates: selected}
if len(selected) != 0 {
selection.outcome = importSelectionHasCandidates
}
return selection, nil
} }
func (s *importTraversal) readGlobMatches(directory, pattern string, location Location) ([]string, []Diagnostic) { func explicitGlobCandidate(access *filesystemAccess, basePath, candidatePath, accessPath string, site importFieldSite, last bool) (selectedImportCandidate, bool, []Diagnostic) {
canonical, err := s.access.canonical(directory) if !access.contains(candidatePath) || !access.contains(accessPath) {
return selectedImportCandidate{}, false, []Diagnostic{*importSelectionDiagnostic(site, CodeImportOutsideRoot, "glob traversal is outside the configuration root")}
}
target, err := access.importEntryTarget(accessPath, nil, true)
if err != nil { if err != nil {
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
return nil, nil return selectedImportCandidate{}, true, nil
} }
return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be canonicalized", location)} return selectedImportCandidate{}, false, []Diagnostic{globCandidateDiagnostic(err, site.location, "glob match could not be inspected")}
} }
if !s.access.contains(canonical) { canonical, err := access.canonical(accessPath)
return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory resolves outside the configuration root", &location)} if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return selectedImportCandidate{}, true, nil
}
return selectedImportCandidate{}, false, []Diagnostic{globFilesystemDiagnostic(err, "glob match could not be canonicalized", site.location)}
} }
if _, err := s.access.relative(canonical); err != nil { if !access.contains(canonical) {
return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)} return selectedImportCandidate{}, false, []Diagnostic{*importSelectionDiagnostic(site, CodeImportOutsideRoot, "glob traversal resolves outside the configuration root")}
} }
var matches []string if !last {
var problem *Diagnostic if !target.dir {
err = s.access.readDirectory(canonical, func(entry fs.DirEntry) bool { return selectedImportCandidate{}, true, nil
s.accounting.entries++
if s.accounting.entries > s.limits.entries {
value := importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)
problem = &value
return false
} }
matched, matchErr := path.Match(pattern, entry.Name()) }
if matchErr != nil { return selectedImportCandidate{
value := importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location) path: normalizedImportCandidatePath(access, candidatePath), accessPath: canonical,
problem = &value label: relativeImportLabel(access, basePath, candidatePath), site: site,
return false }, false, nil
}
func readGlobEntries(access *filesystemAccess, directory string, location Location, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, []Diagnostic) {
target, err := access.importEntryTarget(directory, nil, true)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, true, nil
} }
if matched { return nil, false, []Diagnostic{globCandidateDiagnostic(err, location, "glob directory could not be inspected")}
matches = append(matches, entry.Name()) }
canonical, err := access.canonical(directory)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, true, nil
} }
return true return nil, false, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be canonicalized", location)}
}
if !access.contains(canonical) {
return nil, false, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory resolves outside the configuration root", &location)}
}
if _, err := access.relative(canonical); err != nil {
return nil, false, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)}
}
readPath := target.identity
if readPath == "" {
readPath = canonical
}
entries := make([]fs.DirEntry, 0)
err = visitBoundedImportDirectory(access, readPath, limits.entries, accounting, func(entry fs.DirEntry) error {
entries = append(entries, entry)
return nil
}) })
if problem != nil { if errors.Is(err, errImportDirectoryEntryLimit) {
return nil, []Diagnostic{*problem} return nil, false, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)}
} }
if errors.Is(err, fs.ErrNotExist) { if errors.Is(err, fs.ErrNotExist) {
return nil, nil return nil, true, nil
} }
if err != nil { if err != nil {
return nil, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be traversed", location)} return nil, false, []Diagnostic{globFilesystemDiagnostic(err, "glob directory could not be traversed", location)}
} }
slices.Sort(matches) sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
return matches, nil return entries, false, nil
}
func globCandidateDiagnostic(err error, location Location, message string) Diagnostic {
if errors.Is(err, errImportDirectoryLinkCycle) {
return importDiagnostic(CodeImportCycle, "directory traversal cycle detected", &location)
}
if errors.Is(err, errImportPathOutsideRoot) {
return importDiagnostic(CodeImportOutsideRoot, "glob traversal resolves outside the configuration root", &location)
}
return globFilesystemDiagnostic(err, message, location)
}
func candidatesFromMap(candidates map[string]selectedImportCandidate) []selectedImportCandidate {
paths := make([]string, 0, len(candidates))
for candidatePath := range candidates {
paths = append(paths, candidatePath)
}
slices.Sort(paths)
result := make([]selectedImportCandidate, 0, len(paths))
for _, candidatePath := range paths {
result = append(result, candidates[candidatePath])
}
return result
} }
func validGlobPattern(pattern string) ([]string, bool) { func validGlobPattern(pattern string) ([]string, bool) {

View File

@@ -0,0 +1,203 @@
package engine
import (
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
)
func TestSelectGlobFiltersOnlyWildcardDiscoveredEntries(t *testing.T) {
t.Parallel()
root := t.TempDir()
for name, content := range map[string]string{
".hidden/secret.yaml": "value: hidden\n",
"visible/item.yaml": "value: visible\n",
"visible/source.txt": "value: source\n",
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
for alias, target := range map[string]string{
"visible-alias": "visible",
"selected.yaml": "visible/source.txt",
"unselected.txt": "visible/item.yaml",
".secret.yaml": "visible/source.txt",
} {
if err := os.Symlink(target, filepath.Join(root, filepath.FromSlash(alias))); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
selection := selectGlobForTest(t, access, root, ` - glob: '*/*.yaml'
follow: false
hidden: false
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"visible/item.yaml"}; !equalStrings(got, want) {
t.Fatalf("filtered glob candidates = %#v, want %#v", got, want)
}
selection = selectGlobForTest(t, access, root, ` - glob: '.hidden/*.yaml'
follow: false
hidden: false
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{".hidden/secret.yaml"}; !equalStrings(got, want) {
t.Fatalf("literal hidden component candidates = %#v, want %#v", got, want)
}
selection = selectGlobForTest(t, access, root, ` - glob: 'visible-alias/*.yaml'
follow: false
hidden: false
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"visible-alias/item.yaml"}; !equalStrings(got, want) {
t.Fatalf("literal symlink component candidates = %#v, want %#v", got, want)
}
selection = selectGlobForTest(t, access, root, ` - glob: '*.yaml'
follow: false
hidden: true
`)
if len(selection.candidates) != 0 {
t.Fatalf("follow:false selected a discovered file symlink: %#v", selection.candidates)
}
selection = selectGlobForTest(t, access, root, ` - glob: '*.yaml'
follow: true
hidden: false
`)
if got, want := selectedPathsRelativeTo(selection, root), []string{"selected.yaml"}; !equalStrings(got, want) {
t.Fatalf("filename aliases were not filtered by candidate spelling: %#v, want %#v", got, want)
}
}
func TestSelectGlobAppliesExclusionsBeforeSourceAdmissionWithoutOpeningFiles(t *testing.T) {
t.Parallel()
filesystem := &observedGlobFilesystem{files: fstest.MapFS{
"parts": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"parts/keep.yaml": &fstest.MapFile{Data: []byte("not parsed: [\n")},
"parts/skip-one.yaml": &fstest.MapFile{Data: []byte("not parsed: [\n")},
"parts/skip-two.yaml": &fstest.MapFile{Data: []byte("not parsed: [\n")},
"parts/skip-three.yaml": &fstest.MapFile{Data: []byte("not parsed: [\n")},
}}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, ` - glob: 'parts/*.yaml'
exclude: 'skip-*.yaml'
`)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
limits := defaultImportLimits()
limits.sources = 2
accounting := &importAccounting{sources: 1}
selection, diagnostics := selectGlobImportCandidates(access, ".", specs[0], policy, limits, accounting)
if len(diagnostics) != 0 || !equalStrings(selectedPaths(selection), []string{"parts/keep.yaml"}) {
t.Fatalf("post-exclusion selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if len(filesystem.opened) != 1 || filesystem.opened[0] != "parts" {
t.Fatalf("glob opened paths = %#v, want only the parts directory", filesystem.opened)
}
if accounting.sources+len(selection.candidates) > limits.sources {
t.Fatalf("final candidate set exceeds the private source limit: accounting=%#v selection=%#v", accounting, selection.candidates)
}
}
func TestSelectGlobRejectsEscapingWildcardLinks(t *testing.T) {
t.Parallel()
root := t.TempDir()
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "file.yaml"), []byte("value: outside\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
_, diagnostics := selectGlobForTestWithDiagnostics(t, access, root, ` - glob: '*/file.yaml'
follow: true
`)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportOutsideRoot {
t.Fatalf("escaping glob diagnostics = %#v, want import.outside_root", diagnostics)
}
selection := selectGlobForTest(t, access, root, ` - glob: '*/file.yaml'
follow: false
`)
if len(selection.candidates) != 0 {
t.Fatalf("follow:false selected an escaping alias: %#v", selection.candidates)
}
}
func TestSelectGlobPreservesMatchedDirectoryValidityBeforeExclusion(t *testing.T) {
t.Parallel()
filesystem := fstest.MapFS{
"parts": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"parts/matched.yaml": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
"parts/matched.yaml/x": &fstest.MapFile{Data: []byte("value: x\n")},
}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
_, diagnostics := selectGlobForTestWithDiagnostics(t, access, ".", ` - glob: 'parts/*.yaml'
exclude: '*.yaml'
`)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceNotRegular {
t.Fatalf("excluded directory diagnostics = %#v, want source.not_regular", diagnostics)
}
}
func selectGlobForTest(t *testing.T, access *filesystemAccess, base, declaration string) importSelection {
t.Helper()
selection, diagnostics := selectGlobForTestWithDiagnostics(t, access, base, declaration)
if len(diagnostics) != 0 {
t.Fatalf("glob selection diagnostics = %#v", diagnostics)
}
return selection
}
func selectGlobForTestWithDiagnostics(t *testing.T, access *filesystemAccess, base, declaration string) (importSelection, []Diagnostic) {
t.Helper()
specs, diagnostics := parseImportSpecsYAML(t, declaration)
if len(diagnostics) != 0 {
t.Fatalf("parse diagnostics = %#v", diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
return selectGlobImportCandidates(access, base, specs[0], policy, defaultImportLimits(), &importAccounting{})
}
type observedGlobFilesystem struct {
files fstest.MapFS
opened []string
}
func (filesystem *observedGlobFilesystem) Open(name string) (fs.File, error) {
filesystem.opened = append(filesystem.opened, name)
return filesystem.files.Open(name)
}
func (filesystem *observedGlobFilesystem) Stat(name string) (fs.FileInfo, error) {
return fs.Stat(filesystem.files, name)
}

View File

@@ -1,6 +1,7 @@
package engine package engine
import ( import (
"errors"
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
@@ -106,7 +107,7 @@ func TestLoadGlobImportsRejectInvalidAndUnmatchedPatterns(t *testing.T) {
if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
mainPath := writeGlobRoot(t, directory, test.pattern, false) mainPath := writeGlobRoot(t, directory, test.pattern, test.code == CodeImportMissing)
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code)
}) })
} }
@@ -115,7 +116,7 @@ func TestLoadGlobImportsRejectInvalidAndUnmatchedPatterns(t *testing.T) {
if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil { if err := os.Mkdir(filepath.Join(directory, "empty"), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
mainPath := writeGlobRoot(t, directory, "empty/*.yml", true) mainPath := writeGlobRoot(t, directory, "empty/*.yml", false)
result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
if err != nil || len(result.Sources()) != 1 { if err != nil || len(result.Sources()) != 1 {
t.Fatalf("optional unmatched glob = %#v, %v", result, err) t.Fatalf("optional unmatched glob = %#v, %v", result, err)
@@ -139,7 +140,7 @@ func TestLoadGlobImportsPreserveIdentityAndFileRules(t *testing.T) {
if err := os.MkdirAll(filepath.Join(directory, "conf/matched.yml"), 0o755); err != nil { if err := os.MkdirAll(filepath.Join(directory, "conf/matched.yml"), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
mainPath := writeGlobRoot(t, directory, "conf/*.yml", true) mainPath := writeGlobRoot(t, directory, "conf/*.yml", false)
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourceNotRegular) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourceNotRegular)
}) })
@@ -168,6 +169,10 @@ func TestLoadGlobImportsConfineSymlinkDirectories(t *testing.T) {
mainPath := writeGlobRoot(t, root, "linked/*.yml", false) mainPath := writeGlobRoot(t, root, "linked/*.yml", false)
result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
if err != nil || result.Config.Items["linked"] != "linked" { if err != nil || result.Config.Items["linked"] != "linked" {
var loadError *LoadError
if errors.As(err, &loadError) {
t.Fatalf("contained directory symlink = %#v, diagnostics %#v", result, loadError.Diagnostics())
}
t.Fatalf("contained directory symlink = %#v, %v", result, err) t.Fatalf("contained directory symlink = %#v, %v", result, err)
} }
}) })
@@ -179,7 +184,7 @@ func TestLoadGlobImportsConfineSymlinkDirectories(t *testing.T) {
if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil {
t.Skipf("symlinks unavailable: %v", err) t.Skipf("symlinks unavailable: %v", err)
} }
mainPath := writeGlobRoot(t, root, "linked/*.yml", true) mainPath := writeGlobRoot(t, root, "linked/*.yml", false)
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot)
}) })
} }
@@ -198,7 +203,7 @@ func TestOptionalGlobDoesNotSuppressDirectoryPermissionFailure(t *testing.T) {
_ = opened.Close() _ = opened.Close()
t.Skip("filesystem does not enforce the unreadable-directory fixture") t.Skip("filesystem does not enforce the unreadable-directory fixture")
} }
mainPath := writeGlobRoot(t, directory, "locked/*.yml", true) mainPath := writeGlobRoot(t, directory, "locked/*.yml", false)
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportPermission) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportPermission)
} }
@@ -207,7 +212,7 @@ func TestGlobTraversalBoundsNonmatchesAndAcceptedCandidates(t *testing.T) {
for _, name := range []string{"one.txt", "two.txt", "three.txt"} { for _, name := range []string{"one.txt", "two.txt", "three.txt"} {
writeImportFile(t, directory, "many/"+name, "ignored\n") writeImportFile(t, directory, "many/"+name, "ignored\n")
} }
mainPath := writeGlobRoot(t, directory, "many/*.yml", true) mainPath := writeGlobRoot(t, directory, "many/*.yml", false)
main := parseImportMain(t, mainPath) main := parseImportMain(t, mainPath)
limits := defaultImportLimits() limits := defaultImportLimits()
limits.entries = 2 limits.entries = 2
@@ -263,11 +268,11 @@ func TestFilesystemContextIsValidatedWithoutImportsOrActiveOverlay(t *testing.T)
} }
} }
func writeGlobRoot(t *testing.T, directory, pattern string, optional bool) string { func writeGlobRoot(t *testing.T, directory, pattern string, required bool) string {
t.Helper() t.Helper()
optionalLine := "" requiredLine := ""
if optional { if required {
optionalLine = "\n optional: true" requiredLine = "\n required: true"
} }
return writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - glob: '"+pattern+"'"+optionalLine+"\n") return writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - glob: '"+pattern+"'"+requiredLine+"\n")
} }

View File

@@ -42,15 +42,15 @@ func (c *retainedNodeCounter) add(root *document.Node) bool {
// History applications borrow immutable policy-validated input trees. Metadata // History applications borrow immutable policy-validated input trees. Metadata
// records answer presence and origin queries; replay reconstructs value changes. // records answer presence and origin queries; replay reconstructs value changes.
// Replay reconciles with canonical decoding before domain resolution so both // Replay reconciles with canonical decoding, applies supplied CLI contributions,
// contracts agree on the effective result without retaining a tree per field. // then records domain resolution without retaining a tree per field.
type historyApplication struct { type historyApplication struct {
layer Layer layer Layer
path Path path Path
value *document.Node value *document.Node
origin Origin origin Origin
sequences []sequenceHistoryContribution sequences []sequenceHistoryContribution
sequencesByPath map[string]sequenceHistoryContribution sequencesByPath map[string]sequenceHistoryContributionIndex
} }
func cloneHistoryApplications(values []historyApplication) []historyApplication { func cloneHistoryApplications(values []historyApplication) []historyApplication {
@@ -92,7 +92,14 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef
for index, child := range node.Sequence { for index, child := range node.Sequence {
effectiveIndex := index effectiveIndex := index
if projected { if projected {
effectiveIndex += contribution.offset if contribution.mode == sequenceHistoryMapped {
effectiveIndex = contribution.outputForInput(index)
if effectiveIndex < 0 {
continue
}
} else {
effectiveIndex += contribution.offset
}
} }
walk(child, local.Index(index), effective.Index(effectiveIndex)) walk(child, local.Index(index), effective.Index(effectiveIndex))
} }
@@ -112,7 +119,18 @@ func historyApplicationLocalPath(application historyApplication, effective Path)
for _, segment := range relative.segments { for _, segment := range relative.segments {
if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found { if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found {
index, valid := schemaIndex(segment) index, valid := schemaIndex(segment)
if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length { if !valid || segment.kind == pathSegmentKey {
return Path{}, false
}
if contribution.mode == sequenceHistoryMapped {
inputIndex := contribution.inputForOutput(index)
if inputIndex < 0 {
return Path{}, false
}
local = local.Index(inputIndex)
continue
}
if index < contribution.offset || index >= contribution.offset+contribution.length {
return Path{}, false return Path{}, false
} }
local = local.Index(index - contribution.offset) local = local.Index(index - contribution.offset)
@@ -149,6 +167,16 @@ func applicationOperationPath(application historyApplication, query Path, lookup
return container, true return container, true
} }
} }
indexed, _ := sequenceHistoryAt(application.sequencesByPath, contribution.path)
if indexed.mode == sequenceHistoryMapped && len(query.segments) > len(container.segments) {
outputIndex, valid := schemaIndex(query.segments[len(container.segments)])
_, replaced := indexed.replacedOutputs[outputIndex]
if valid && replaced {
if _, present := documentAtPathWithLookup(application.value, local, lookup); !ok || !present {
return container.Index(outputIndex), true
}
}
}
if !ok { if !ok {
continue continue
} }
@@ -206,14 +234,14 @@ func applyHistoryApplication(root **document.Node, application historyApplicatio
mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath) mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath)
return return
} }
mergeHistoryAtPath(root, application.path, application.value, index) mergeHistoryAtPath(root, application.path, application.value, index, application.sequencesByPath)
} }
func mergeHistoryDocument(target **document.Node, incoming *document.Node, index *mappingIndex) { func mergeHistoryDocument(target **document.Node, incoming *document.Node, index *mappingIndex) {
mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil) mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil)
} }
func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) { func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContributionIndex) {
if incoming == nil { if incoming == nil {
return return
} }
@@ -238,23 +266,53 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node,
} }
} }
func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) { func sequenceHistoryAt(contributions map[string]sequenceHistoryContributionIndex, path Path) (sequenceHistoryContributionIndex, bool) {
contribution, found := contributions[path.String()] contribution, found := contributions[path.String()]
return contribution, found return contribution, found
} }
func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution { type sequenceHistoryContributionIndex struct {
sequenceHistoryContribution
projection sequenceItemProjectionIndex
replacedOutputs map[int]struct{}
}
func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContributionIndex {
if len(values) == 0 { if len(values) == 0 {
return nil return nil
} }
index := make(map[string]sequenceHistoryContribution, len(values)) index := make(map[string]sequenceHistoryContributionIndex, len(values))
for _, value := range values { for _, value := range values {
index[value.path.String()] = value contribution := sequenceHistoryContributionIndex{sequenceHistoryContribution: value}
if value.mode == sequenceHistoryMapped {
contribution.projection = newSequenceItemProjectionIndex(value.items)
contribution.replacedOutputs = make(map[int]struct{}, len(value.replacements))
for _, replacement := range value.replacements {
contribution.replacedOutputs[replacement.outputIndex] = struct{}{}
}
}
index[value.path.String()] = contribution
} }
return index return index
} }
func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) { func (contribution sequenceHistoryContributionIndex) outputForInput(inputIndex int) int {
outputIndex, ok := contribution.projection.outputForInput(inputIndex)
if !ok {
return -1
}
return outputIndex
}
func (contribution sequenceHistoryContributionIndex) inputForOutput(outputIndex int) int {
inputIndex, ok := contribution.projection.inputForOutput(outputIndex)
if !ok {
return -1
}
return inputIndex
}
func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContributionIndex) {
if incoming == nil || incoming.Kind != document.Sequence { if incoming == nil || incoming.Kind != document.Sequence {
return return
} }
@@ -262,6 +320,18 @@ func applySequenceHistory(target **document.Node, incoming *document.Node, contr
*target = document.Clone(incoming) *target = document.Clone(incoming)
return return
} }
if contribution.mode == sequenceHistoryMapped {
for _, projection := range contribution.items {
if projection.inputIndex < 0 || projection.inputIndex >= len(incoming.Sequence) || projection.outputIndex < 0 {
continue
}
for len((*target).Sequence) <= projection.outputIndex {
(*target).Sequence = append((*target).Sequence, nil)
}
(*target).Sequence[projection.outputIndex] = document.Clone(incoming.Sequence[projection.inputIndex])
}
return
}
for _, item := range incoming.Sequence { for _, item := range incoming.Sequence {
(*target).Sequence = append((*target).Sequence, document.Clone(item)) (*target).Sequence = append((*target).Sequence, document.Clone(item))
} }
@@ -275,13 +345,15 @@ func cloneSequenceHistoryContributions(values []sequenceHistoryContribution) []s
for index, value := range values { for index, value := range values {
cloned[index] = value cloned[index] = value
cloned[index].path = value.path.clone() cloned[index].path = value.path.clone()
cloned[index].items = append([]sequenceItemProjection(nil), value.items...)
cloned[index].replacements = append([]sequenceItemReplacement(nil), value.replacements...)
} }
return cloned return cloned
} }
func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node, index *mappingIndex) { func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node, index *mappingIndex, sequences map[string]sequenceHistoryContributionIndex) {
if root == nil || *root == nil || len(path.segments) == 0 { if root == nil || *root == nil || len(path.segments) == 0 {
mergeHistoryDocument(root, incoming, index) mergeHistoryDocumentAtPath(root, incoming, Path{}, index, sequences)
return return
} }
current := *root current := *root
@@ -292,7 +364,7 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
return return
} }
if segmentIndex == len(path.segments)-1 { if segmentIndex == len(path.segments)-1 {
mergeHistoryDocument(&current.Sequence[position], incoming, index) mergeHistoryDocumentAtPath(&current.Sequence[position], incoming, Path{}, index, sequences)
return return
} }
current = current.Sequence[position] current = current.Sequence[position]
@@ -306,7 +378,7 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
if position < 0 { if position < 0 {
index.append(current, document.Entry{Key: segment.key, Value: document.Clone(incoming)}) index.append(current, document.Entry{Key: segment.key, Value: document.Clone(incoming)})
} else { } else {
mergeHistoryDocument(&current.Mapping[position].Value, incoming, index) mergeHistoryDocumentAtPath(&current.Mapping[position].Value, incoming, Path{}, index, sequences)
} }
return return
} }

View File

@@ -10,7 +10,10 @@ import (
) )
func TestSequenceHistoryReplaysAppendAtAssignedIndexes(t *testing.T) { func TestSequenceHistoryReplaysAppendAtAssignedIndexes(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id", "/other": "id"}} policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("other")}, identityFields: []string{"id"}},
})
first := parseCompositionDocument(t, "source-1", "records: [{id: a, value: first}]\nother: [{id: x}]\n") first := parseCompositionDocument(t, "source-1", "records: [{id: a, value: first}]\nother: [{id: x}]\n")
second := parseCompositionDocument(t, "source-2", "records: [{id: b, value: second}]\nother: [{id: y}]\n") second := parseCompositionDocument(t, "source-2", "records: [{id: b, value: second}]\nother: [{id: y}]\n")
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second) composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
@@ -75,8 +78,76 @@ func TestSequenceHistoryReplaysAppendAtAssignedIndexes(t *testing.T) {
} }
} }
func TestTierSequenceHistoryReplaysSelectedOperation(t *testing.T) {
t.Parallel()
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"},
}})
for _, test := range []struct {
name string
mode sequenceMode
}{{"append", sequenceModeAppend}, {"upsert", sequenceModeUpsert}, {"replace", sequenceModeReplace}} {
t.Run(test.name, func(t *testing.T) {
mode := test.mode
lower, diagnostics := composeDocumentsWithPolicy(policy, parseCompositionDocument(t, "lower", "records: [{id: a, removed: old}, {id: b, value: old, removed: old}]\n"))
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
secondID := "d"
if mode == sequenceModeUpsert {
secondID = "b"
}
declarations := []sequenceDeclarationSpec{{destination: Keys("records"), modePresent: true, mode: mode}}
higher, diagnostics := composeDocumentsWithCollectionPolicy(policy, lower.collectionPolicies, declarations,
parseCompositionDocument(t, "first-peer", "records: [{id: c, value: first}]\n"),
parseCompositionDocument(t, "second-peer", "records: [{id: "+secondID+", value: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
assembly := newAssemblyAccumulator(lower)
if diagnostics := assembly.applyOverlayWithCollectionPolicy(higher.root, higher.origins, higher.collectionPolicies, higher.applications, LayerBase, policy, declarations); len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
if len(assembly.current.applications) != 3 {
t.Fatalf("source applications = %d, want all three sources", len(assembly.current.applications))
}
if mode == sequenceModeUpsert {
query := Keys("records").Index(1).Key("value")
if _, relevant := applicationOperationPath(assembly.current.applications[1], query, nil); relevant {
t.Fatal("first peer was attributed to another peer's upserted record")
}
origin := historyApplicationOrigin(assembly.current.applications[2], query, nil)
if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "second-peer" {
t.Fatalf("upsert history source = %#v", origin)
}
}
working := parseCompositionDocument(t, "initial", "{}\n")
index := new(mappingIndex)
for _, application := range assembly.current.applications {
applyHistoryApplication(&working, application, index)
}
if !configurationValuesEqual(working, assembly.current.root) {
t.Fatal("tier history differs from structural assembly before decoding")
}
overlay := parseCompositionDocument(t, "overlay", "records: [{id: c, value: overlay}]\n")
declarations[0].mode = sequenceModeUpsert
if diagnostics := assembly.applyOverlayWithCollectionPolicy(overlay, nil, collectionPolicyScope{}, nil, LayerOverlay, policy, declarations); len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
applyHistoryApplication(&working, assembly.current.applications[3], index)
if !configurationValuesEqual(working, assembly.current.root) {
t.Fatal("subsequent overlay history differs from structural assembly")
}
})
}
}
func TestSequenceHistoryPreservesTierReplacementAndEmptyLists(t *testing.T) { func TestSequenceHistoryPreservesTierReplacementAndEmptyLists(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id", "/independent": "id"}} policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("independent")}, identityFields: []string{"id"}},
})
first, diagnostics := composeDocumentsWithPolicy(policy, first, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "underlay-1a", "records: [{id: shared}]\nindependent: [{id: x}]\n"), parseCompositionDocument(t, "underlay-1a", "records: [{id: shared}]\nindependent: [{id: x}]\n"),
parseCompositionDocument(t, "underlay-1b", "records: [{id: lower}]\n"), parseCompositionDocument(t, "underlay-1b", "records: [{id: lower}]\n"),
@@ -145,10 +216,53 @@ func TestHistoryMergeSupportsSemanticIndexedPrefixes(t *testing.T) {
if !present || value.Text != "new" { if !present || value.Text != "new" {
t.Fatalf("indexed helper replay = %#v, %v", value, present) t.Fatalf("indexed helper replay = %#v, %v", value, present)
} }
root = parseCompositionDocument(t, "base", "records: [{id: one, helper: {items: [{id: a}]}}]\n")
application.value = parseCompositionDocument(t, "helper", "items: [{id: b}]\n")
application.sequences = []sequenceHistoryContribution{{path: Keys("items"), mode: sequenceHistoryAppend, offset: 1, length: 1}}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
applyHistoryApplication(&root, application, new(mappingIndex))
items, present := documentAtPath(root, Keys("records").Index(0).Key("helper").Key("items"))
if !present || len(items.Sequence) != 2 {
t.Fatalf("indexed helper lost its local sequence operation: %#v", items)
}
}
func TestLayeredUpsertExplanationFollowsStructuralOrder(t *testing.T) {
t.Parallel()
type config struct {
Records []map[string]any `yaml:"records"`
}
loaded, err := testGroupedLoad(loadRequest[config]{
main: loadInput{source: Bytes("main", []byte("configkit:\n sequences: [{destination: [records], key: id}]\nrecords: [{id: b, value: new-b}, {id: c, value: new-c}, {id: a, value: new-a}]\n"))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("records: [{id: a, value: old-a}, {id: b, value: old-b, removed: old}]\n"))}}}},
})
if err != nil {
t.Fatal(err)
}
explanation, ok := loaded.Explain(Keys("records"))
if !ok || len(explanation.History) == 0 {
t.Fatal("missing collection history")
}
for _, change := range explanation.History {
if change.Origin.Operation == ChangeDecoding {
t.Fatal("structural upsert was misreported as a decoding reconciliation")
}
}
if last := explanation.History[len(explanation.History)-1]; last.After.YAML != explanation.Effective.YAML || last.Origin.Layer != LayerBase {
t.Fatalf("main contribution did not produce the effective collection: %#v", last)
}
removed, ok := loaded.Explain(Keys("records").Index(1).Key("removed"))
if !ok || len(removed.History) == 0 || removed.Effective.Present {
t.Fatalf("missing displaced field history: %#v, %v", removed, ok)
}
if last := removed.History[len(removed.History)-1]; last.After.Present || last.Origin.Layer != LayerBase || last.Origin.Operation == ChangeDecoding {
t.Fatalf("upsert removal attribution = %#v", last)
}
} }
func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T) { func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}} policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy, composed, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"), parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
parseCompositionDocument(t, "second", "records: [{id: second, token: private}]"), parseCompositionDocument(t, "second", "records: [{id: second, token: private}]"),
@@ -165,7 +279,7 @@ func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T
inspection := &Inspection{ inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: composed.root, final: composed.root, applications: composed.applications, input: composed.root, preCLI: composed.root, final: composed.root, applications: composed.applications,
} }
inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
return reflect.DeepEqual(candidate, path) return reflect.DeepEqual(candidate, path)
@@ -193,7 +307,7 @@ func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T
} }
func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) { func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}} policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy, composed, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"), parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
parseCompositionDocument(t, "second", "records: [{id: second, token: PRIVATE-RECORD-CANARY}]"), parseCompositionDocument(t, "second", "records: [{id: second, token: PRIVATE-RECORD-CANARY}]"),
@@ -212,7 +326,7 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
finalizeMetadataPaths(&metadata, final, applications) finalizeMetadataPaths(&metadata, final, applications)
inspection := &Inspection{ inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: final, final: final, applications: applications, input: final, preCLI: final, final: final, applications: applications,
} }
inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
return reflect.DeepEqual(candidate, path) return reflect.DeepEqual(candidate, path)
@@ -230,6 +344,241 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
} }
} }
func TestMappedUpsertProjectsSensitiveReplacementEvidence(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
- id: middle
- id: last
token: MAPPED-SECRET-CANARY
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
token: visible
- id: first
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher,
sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
secretPath := Keys("records").Index(2).Key("token")
secretNode, present := documentAtPath(lowerRoot, Keys("records").Index(2).Key("token"))
if !present {
t.Fatal("sensitive source token is missing")
}
secretNode.Sensitive = true
application := historyApplication{
layer: LayerOverlay, value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
applications := []historyApplication{{value: lowerRoot}, application}
replayed := &document.Node{Kind: document.Mapping}
for _, current := range applications {
applyHistoryApplication(&replayed, current, new(mappingIndex))
}
replayedRecords, _ := documentAtPath(replayed, Keys("records"))
if !reflect.DeepEqual(replayedRecords.Sequence, combined.value.Sequence) {
t.Fatalf("replayed value = %#v, want combination = %#v", replayedRecords.Sequence, combined.value.Sequence)
}
finalRoot := document.Clone(lowerRoot)
finalRecords, _ := documentAtPath(finalRoot, Keys("records"))
finalRecords.Sequence = document.Clone(combined.value).Sequence
var metadata Metadata
finalizeMetadataPaths(&metadata, finalRoot, applications)
inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: finalRoot, preCLI: finalRoot, final: finalRoot, applications: applications,
}
secretRule := func(candidate Path) bool { return reflect.DeepEqual(candidate, secretPath) }
inspection.secrets = classifySecretPaths(nil, secretRule, metadata, finalRoot, applications)
explanation, ok := inspection.Explain(secretPath)
if !ok || !explanation.Effective.Redacted {
t.Fatalf("mapped replacement explanation = %#v, %v", explanation, ok)
}
var rendered bytes.Buffer
if err := inspection.WriteExplanation(&rendered, secretPath); err != nil {
t.Fatal(err)
}
if strings.Contains(rendered.String(), "MAPPED-SECRET-CANARY") || !strings.Contains(rendered.String(), "[REDACTED]") {
t.Fatalf("mapped replacement history leaked or was not redacted: %s", rendered.String())
}
}
func TestSequenceHistoryReplaysCombinationEvidence(t *testing.T) {
tests := []struct {
name string
operation sequenceMode
lower string
higher string
}{
{
name: "append",
operation: sequenceModeAppend,
lower: "records: [{id: first}, {id: last}]\n",
higher: "records: [{id: added}]\n",
},
{
name: "upsert",
operation: sequenceModeUpsert,
lower: `records:
- id: first
label: same
nested: {removed: lower}
- id: middle
- id: last
nested: {removed: lower}
`,
higher: `records:
- id: last
nested: {kept: higher}
- id: first
label: same
- id: added
`,
},
{
name: "replace",
operation: sequenceModeReplace,
lower: "records: [{id: discarded}]\n",
higher: "records: [{id: replacement}]\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", test.lower)
higherRoot := parseCompositionDocument(t, "higher", test.higher)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
fields := []string(nil)
if test.operation != sequenceModeReplace {
fields = []string{"id"}
}
combined, diagnostics := combineSequence(lower, higher, sequenceCombinationPolicy(test.operation, fields...), sequenceCombinationContext{destination: Keys("records")})
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
mode := sequenceHistoryMapped
if test.operation == sequenceModeReplace {
mode = sequenceHistoryReplace
}
length := 0
if mode == sequenceHistoryReplace {
length = len(combined.higherItems)
}
application := historyApplication{
layer: LayerOverlay,
value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: mode, length: length,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
replayed := document.Clone(lowerRoot)
applyHistoryApplication(&replayed, application, new(mappingIndex))
got, present := documentAtPath(replayed, Keys("records"))
if !present || !reflect.DeepEqual(got.Sequence, combined.value.Sequence) {
t.Fatalf("replayed sequence = %#v, want combination = %#v", got, combined.value.Sequence)
}
})
}
}
func TestMappedSequenceHistoryReplacesWholeRecordsAndKeepsDisplacedEvidence(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
label: same
- id: middle
- id: last
nested: {removed: LOWER-CANARY}
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
nested: {kept: higher}
- id: first
label: same
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher,
sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
application := historyApplication{
layer: LayerOverlay, value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
clonedApplications := cloneHistoryApplications([]historyApplication{application})
clonedApplications[0].sequences[0].items[0].outputIndex = 99
clonedApplications[0].sequences[0].replacements[0].outputIndex = 99
if application.sequences[0].items[0].outputIndex == 99 || application.sequences[0].replacements[0].outputIndex == 99 {
t.Fatal("cloned mapped history aliases its source projections")
}
query := Keys("records").Index(2).Key("nested").Key("removed")
operationPath, relevant := applicationOperationPath(application, query, nil)
if !relevant || operationPath.String() != "/records/2" {
t.Fatalf("removed descendant operation = %s, %v", operationPath.String(), relevant)
}
localPath, ok := historyApplicationLocalPath(application, Keys("records").Index(2).Key("nested").Key("kept"))
if !ok || localPath.String() != "/records/0/nested/kept" {
t.Fatalf("mapped effective path localizes to %s, %v", localPath.String(), ok)
}
origin := historyApplicationOrigin(application, operationPath, nil)
if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "higher" {
t.Fatalf("replacement operation origin = %#v", origin)
}
var walkedMappedPath bool
walkHistoryApplication(application, func(local, effective Path, _ *document.Node) {
if local.String() == "/records/0/nested/kept" && effective.String() == "/records/2/nested/kept" {
walkedMappedPath = true
}
})
if !walkedMappedPath {
t.Fatal("history walk did not project a noncontiguous incoming item")
}
replayed := document.Clone(lowerRoot)
applyHistoryApplication(&replayed, application, new(mappingIndex))
sequence, _ := documentAtPath(replayed, Keys("records"))
if _, present := documentAtPath(sequence.Sequence[2], Keys("nested", "removed")); present {
t.Fatal("whole-record replacement retained an omitted descendant")
}
if value, present := documentAtPath(sequence.Sequence[2], Keys("nested", "kept")); !present || value.Text != "higher" {
t.Fatalf("replacement descendant = %#v, %v", value, present)
}
if sequence.Sequence[1].Mapping[0].Value.Text != "middle" {
t.Fatal("unchanged neighboring record moved or changed")
}
if value, present := documentAtPath(lowerRoot, Keys("records").Index(2).Key("nested").Key("removed")); !present || value.Text != "LOWER-CANARY" {
t.Fatalf("displaced lower contribution was not retained as raw history: %#v, %v", value, present)
}
if got := application.sequences[0].replacements; !reflect.DeepEqual(got, combined.replacements) || len(got) != 2 {
t.Fatalf("replacement evidence lost equal-valued or noncontiguous events: %#v", got)
}
}
func mustParsePath(t *testing.T, value string) Path { func mustParsePath(t *testing.T, value string) Path {
t.Helper() t.Helper()
path, err := ParsePath(value) path, err := ParsePath(value)

View File

@@ -0,0 +1,260 @@
package engine
import (
"errors"
"path/filepath"
"reflect"
"testing"
)
func TestOptionalEmptyImportSelectionsRemainSourceLocalAndDetached(t *testing.T) {
t.Parallel()
type config struct {
Value string `yaml:"value"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- dir: absent
- path: missing.yml
- path: []
- dir: fragments
exclude: '*.yml'
- path: [partial-missing.yml, present.yml]
- path: []
required: true
`)
writeImportFile(t, directory, "fragments/ignored.yml", "ignored: true\n")
writeImportFile(t, directory, "present.yml", "value: imported\n")
result, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
if err != nil {
t.Fatal(err)
}
if result.Config.Value != "imported" {
t.Fatalf("value = %q", result.Config.Value)
}
sources := result.Sources()
if len(sources) != 2 {
t.Fatalf("sources = %#v", sources)
}
wantPaths := []string{"/configkit/imports/0", "/configkit/imports/1", "/configkit/imports/2", "/configkit/imports/3"}
wantSelectors := []importSelector{importSelectorDirectory, importSelectorPath, importSelectorPath, importSelectorDirectory}
evidence := sources[0].importSelections
if len(evidence) != len(wantPaths) {
t.Fatalf("empty-selection evidence = %#v", evidence)
}
for index, selection := range evidence {
if selection.path.String() != wantPaths[index] || selection.location.SourceID != sources[0].ID ||
selection.selector != wantSelectors[index] || selection.outcome != importSelectionNone {
t.Errorf("selection %d = %#v", index, selection)
}
}
if len(sources[1].importSelections) != 0 {
t.Fatalf("imported source unexpectedly owns declaration evidence: %#v", sources[1])
}
sources[0].importSelections[0].path.segments[0].key = "changed"
sources[0].importSelections[0].location.SourceID = "changed"
if got := result.Sources()[0].importSelections[0]; got.path.String() != wantPaths[0] || got.location.SourceID != "source-1" {
t.Fatalf("mutating a query snapshot changed the retained record: %#v", got)
}
other, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(other.Sources()[0].importSelections, result.Inspection.Sources()[0].importSelections) {
t.Fatal("one load's source snapshot affected another load")
}
}
func TestFailedLoadRetainsOnlyCompletedEmptyImportSelections(t *testing.T) {
t.Parallel()
type config struct {
Value string `yaml:"value"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- glob: absent/*.yml
- path: required-missing.yml
required: true
- dir: never-visited
value: main
`)
_, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
var loadError *LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %T %v", err, err)
}
sources := loadError.Sources()
if len(sources) != 1 || len(sources[0].importSelections) != 1 {
t.Fatalf("failed-load sources = %#v", sources)
}
selection := sources[0].importSelections[0]
if selection.path.String() != "/configkit/imports/0" || selection.location.SourceID != sources[0].ID || selection.selector != importSelectorGlob || selection.outcome != importSelectionNone {
t.Fatalf("retained selection = %#v", selection)
}
sources[0].importSelections[0].path.segments[0].key = "changed"
if got := loadError.Sources()[0].importSelections[0].path.String(); got != "/configkit/imports/0" {
t.Fatalf("mutating failed-load snapshot changed retained evidence: %q", got)
}
}
func TestImportSelectionEvidenceSurvivesCompositionUnderlaysAndResolutionEntry(t *testing.T) {
t.Parallel()
type item struct {
ID string `yaml:"id"`
Metadata map[string]string `yaml:"metadata"`
}
type config struct {
Value string `yaml:"value"`
Helper string `yaml:"helper"`
Items []item `yaml:"items"`
Overlay string `yaml:"overlay"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: helper.yml
- path: item.yml
destination: [items]
placement: sequence-item
- dir: absent-main
value: main
`)
helperPath := writeImportFile(t, directory, "helper.yml", `configkit:
imports:
- glob: absent-helper/*.yml
helper: value
`)
itemPath := writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: []
- path: item-helper.yml
destination: [metadata]
id: selected
`)
itemHelperPath := writeImportFile(t, directory, "item-helper.yml", `configkit:
imports:
- dir: absent-item-helper
key: value
`)
underlayPath := writeImportFile(t, directory, "underlay.yml", `configkit:
imports:
- path: []
value: underlay
`)
overlayPath := writeImportFile(t, directory, filepath.Join("overlays", "selected.yml"), "overlay: value\n")
var resolutionSources map[string]SourceInfo
loaded, err := LoadLayered(Input{Source: File(mainPath), Imports: true}, testSecretBinding(), LayeredOptions[config]{
Definition: Definition[config]{Resolve: func(_ *config, resolution *Resolution) error {
resolutionSources = make(map[string]SourceInfo)
for _, id := range []string{"source-1", "source-2", "source-3", "source-4", "source-5", "source-6"} {
if source, ok := resolution.Source(id); ok {
resolutionSources[id] = source
}
}
return nil
}},
Underlays: []Underlay{{Inputs: []Input{{Source: File(underlayPath), Imports: true}}}},
Overlay: "selected", OverlayFilesystem: &FilesystemContext{BaseDir: directory, Root: directory},
})
if err != nil {
if loadError, ok := err.(*LoadError); ok {
t.Fatalf("LoadLayered diagnostics: %#v", loadError.Diagnostics())
}
t.Fatalf("LoadLayered: %v", err)
}
if loaded.Config.Value != "main" || loaded.Config.Overlay != "value" || len(loaded.Config.Items) != 1 || loaded.Config.Items[0].ID != "selected" {
t.Fatalf("configuration = %#v", loaded.Config)
}
want := map[string]string{
"source-1": "/configkit/imports/2",
"source-2": "/configkit/imports/0",
"source-3": "/configkit/imports/0",
"source-4": "/configkit/imports/0",
"source-5": "/configkit/imports/0",
}
sources := loaded.Inspection.Sources()
if len(sources) != 6 {
t.Fatalf("sources = %#v", sources)
}
if sources[1].Path != canonicalTestPath(t, helperPath) || sources[2].Path != canonicalTestPath(t, itemPath) ||
sources[3].Path != canonicalTestPath(t, itemHelperPath) || sources[4].Path != canonicalTestPath(t, underlayPath) {
t.Fatalf("acquired source order = %#v", sources)
}
if sources[3].destinationUnresolved || sources[3].Destination.String() != "/items/0/metadata" {
t.Fatalf("nested item helper destination = %#v", sources[3])
}
if sources[5].Path != canonicalTestPath(t, overlayPath) || sources[5].Layer != LayerOverlay {
t.Fatalf("overlay source = %#v", sources[5])
}
for id, path := range want {
var found *SourceInfo
for index := range sources {
if sources[index].ID == id {
found = &sources[index]
break
}
}
if found == nil || len(found.importSelections) != 1 || found.importSelections[0].path.String() != path || found.importSelections[0].location.SourceID != id {
t.Errorf("source %s evidence = %#v, want %s", id, found, path)
}
if callbackSource, ok := resolutionSources[id]; !ok || len(callbackSource.importSelections) != 1 || callbackSource.importSelections[0].path.String() != path {
t.Errorf("resolution source %s evidence = %#v, want %s", id, callbackSource, path)
}
}
if len(sources[5].importSelections) != 0 {
t.Fatalf("overlay unexpectedly has import-selection evidence: %#v", sources[5])
}
}
func TestFailedNestedItemLoadKeepsEvidenceWithUnresolvedDestination(t *testing.T) {
t.Parallel()
type item struct {
ID string `yaml:"id"`
}
type config struct {
Items []item `yaml:"items"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: helper.yml
destination: [metadata]
`)
writeImportFile(t, directory, "helper.yml", `configkit:
imports:
- dir: absent
- path: missing.yml
required: true
key: value
`)
_, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
var loadError *LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %T %v", err, err)
}
for _, source := range loadError.Sources() {
if source.Path != canonicalTestPath(t, filepath.Join(directory, "helper.yml")) {
continue
}
if !source.destinationUnresolved || source.Destination.String() != "/items" || len(source.importSelections) != 1 ||
source.importSelections[0].path.String() != "/configkit/imports/0" || source.importSelections[0].location.SourceID != source.ID {
t.Fatalf("failed nested item source = %#v", source)
}
return
}
t.Fatalf("helper source missing from failed-load snapshot: %#v", loadError.Sources())
}

View File

@@ -0,0 +1,325 @@
package engine
import (
"fmt"
"path/filepath"
"strings"
)
type importNodeKey struct {
namespaceKind string
namespaceAnchor string
resourceKind string
resourcePath string
}
func (key importNodeKey) dependencyKey() dependencyKey {
return dependencyKey{key.namespaceKind, key.namespaceAnchor, key.resourceKind, key.resourcePath}
}
type importContributionRole string
const (
importRoleMain importContributionRole = "main"
importRoleMapping importContributionRole = "mapping-import"
importRoleItem importContributionRole = "item-import"
)
type importGraphNode struct {
key importNodeKey
canonical string
sourceID string
role importContributionRole
}
type importInclusionEdge struct {
importer importNodeKey
imported importNodeKey
location Location
}
// importInclusionGraph retains source inclusion independently of mapping-scope
// ownership. Scheduling edges point from each imported prerequisite to its
// importer, matching the shared dependency scheduler's convention.
type importInclusionGraph struct {
nodes []importGraphNode
edges []importInclusionEdge
index map[importNodeKey]int
}
func newImportInclusionGraph() importInclusionGraph {
return importInclusionGraph{index: make(map[importNodeKey]int)}
}
func (graph *importInclusionGraph) addNode(node importGraphNode) int {
if graph.index == nil {
graph.index = make(map[importNodeKey]int)
}
if index, exists := graph.index[node.key]; exists {
return index
}
index := len(graph.nodes)
graph.index[node.key] = index
graph.nodes = append(graph.nodes, node)
return index
}
func (graph *importInclusionGraph) addEdge(importer, imported importNodeKey, location Location) int {
index := len(graph.edges)
graph.edges = append(graph.edges, importInclusionEdge{
importer: importer, imported: imported, location: location,
})
return index
}
func (graph importInclusionGraph) clone() importInclusionGraph {
cloned := importInclusionGraph{
nodes: append([]importGraphNode(nil), graph.nodes...),
edges: make([]importInclusionEdge, len(graph.edges)),
index: make(map[importNodeKey]int, len(graph.index)),
}
for index, edge := range graph.edges {
cloned.edges[index] = importInclusionEdge{
importer: edge.importer, imported: edge.imported, location: edge.location,
}
}
for key, index := range graph.index {
cloned.index[key] = index
}
return cloned
}
func mergeImportInclusionGraph(target *importInclusionGraph, source importInclusionGraph) {
for _, node := range source.nodes {
target.addNode(node)
}
for _, edge := range source.edges {
target.addEdge(edge.importer, edge.imported, edge.location)
}
}
func (graph importInclusionGraph) scheduleCycle(closingEdge int, location Location) (Diagnostic, bool) {
keys := make([]dependencyKey, len(graph.nodes))
for index, node := range graph.nodes {
keys[index] = node.key.dependencyKey()
}
edges := make([]dependencyEdge, 0, len(graph.edges))
for _, edge := range graph.edges {
importer, importerOK := graph.index[edge.importer]
imported, importedOK := graph.index[edge.imported]
if !importerOK || !importedOK {
continue
}
edges = append(edges, dependencyEdge{prerequisite: imported, dependent: importer})
}
_, cycle := scheduleDependencies(keys, edges)
if len(cycle) < 2 {
return Diagnostic{}, false
}
// Scheduler edges reverse inclusion edges. Reverse its closed cycle to
// render the authored import route while keeping the canonical start node.
route := make([]int, 0, len(cycle))
route = append(route, cycle[0])
for index := len(cycle) - 2; index > 0; index-- {
route = append(route, cycle[index])
}
route = append(route, cycle[0])
paths := make([]string, len(route))
for index, node := range route {
paths[index] = graph.nodes[node].canonical
}
related := make([]Location, 0, len(route)-2)
for index := 0; index+1 < len(route); index++ {
importerKey := graph.nodes[route[index]].key
importedKey := graph.nodes[route[index+1]].key
edgeIndex := -1
for candidate, edge := range graph.edges {
if edge.importer == importerKey && edge.imported == importedKey {
edgeIndex = candidate
if candidate == closingEdge {
break
}
}
}
if edgeIndex == closingEdge {
continue
}
if edgeIndex >= 0 {
related = append(related, graph.edges[edgeIndex].location)
}
}
return Diagnostic{
Code: CodeImportCycle, Severity: SeverityError,
Message: "import cycle detected: " + strings.Join(paths, " -> "),
Primary: cloneLocation(&location), Related: related,
}, true
}
func scheduleAcquisitionScope(scope acquisitionScope, graph importInclusionGraph) acquisitionScope {
slots := make([]int, 0, len(scope.contributions))
for index, contribution := range scope.contributions {
if contribution.scopeOwner != scope.owner {
continue
}
slots = append(slots, index)
}
scheduled := scope
scheduled.contributions = scheduleContributionSlots(scope.contributions, slots, graph)
return scheduled
}
func scheduleAcquisitionGroup(scopes []acquisitionScope, graph importInclusionGraph) acquisitionScope {
var contributions []acquiredContribution
for _, scope := range scopes {
contributions = append(contributions, scope.contributions...)
}
slots := make([]int, len(contributions))
for index := range slots {
slots[index] = index
}
return acquisitionScope{contributions: scheduleContributionSlots(contributions, slots, graph)}
}
func scheduleContributionSlots(contributions []acquiredContribution, slots []int, graph importInclusionGraph) []acquiredContribution {
if len(slots) < 2 {
return append([]acquiredContribution(nil), contributions...)
}
positions := make(map[importNodeKey]int, len(slots))
keys := make([]dependencyKey, len(slots))
for position, slot := range slots {
contribution := contributions[slot]
positions[contribution.identity] = position
keys[position] = contribution.identity.dependencyKey()
}
edges := make([]dependencyEdge, 0, len(graph.edges))
for _, edge := range graph.edges {
importer, importerOK := positions[edge.importer]
imported, importedOK := positions[edge.imported]
if importerOK && importedOK {
edges = append(edges, dependencyEdge{prerequisite: imported, dependent: importer})
}
}
order, _ := scheduleDependencies(keys, edges)
if len(order) != len(keys) {
return append([]acquiredContribution(nil), contributions...)
}
scheduled := append([]acquiredContribution(nil), contributions...)
for position, scheduledIndex := range order {
scheduled[slots[position]] = contributions[slots[scheduledIndex]]
}
return scheduled
}
type importNamespaceAnchors struct {
kind string
host string
native string
input string
}
func explicitNamespaceAnchor(group, input int) string {
return fmt.Sprintf("%020d/%020d", group, input)
}
func reanchorImportKey(key importNodeKey, anchors importNamespaceAnchors) importNodeKey {
if anchors.kind != "" {
key.namespaceKind = anchors.kind
}
switch {
case key.resourceKind == "synthetic":
key.namespaceAnchor = anchors.input
case key.namespaceKind == "native":
key.namespaceAnchor = anchors.native
default:
key.namespaceAnchor = anchors.host
}
return key
}
func reanchorImportGraph(graph importInclusionGraph, anchors importNamespaceAnchors) (importInclusionGraph, map[importNodeKey]importNodeKey) {
reanchored := newImportInclusionGraph()
keys := make(map[importNodeKey]importNodeKey, len(graph.nodes))
for _, node := range graph.nodes {
old := node.key
node.key = reanchorImportKey(old, anchors)
keys[old] = node.key
reanchored.addNode(node)
}
for _, edge := range graph.edges {
reanchored.addEdge(keys[edge.importer], keys[edge.imported], edge.location)
}
return reanchored, keys
}
func reanchorAcquisitionScope(scope *acquisitionScope, keys map[importNodeKey]importNodeKey, anchors importNamespaceAnchors) {
if scope == nil {
return
}
if key, exists := keys[scope.owner]; exists {
scope.owner = key
} else {
scope.owner = reanchorImportKey(scope.owner, anchors)
}
for index := range scope.contributions {
contribution := &scope.contributions[index]
if key, exists := keys[contribution.identity]; exists {
contribution.identity = key
} else {
contribution.identity = reanchorImportKey(contribution.identity, anchors)
}
if key, exists := keys[contribution.scopeOwner]; exists {
contribution.scopeOwner = key
} else {
contribution.scopeOwner = reanchorImportKey(contribution.scopeOwner, anchors)
}
reanchorAcquisitionScope(contribution.pendingItem, keys, anchors)
if contribution.nested != nil {
for nestedIndex := range contribution.nested.acquisition {
reanchorAcquisitionScope(&contribution.nested.acquisition[nestedIndex], keys, anchors)
}
}
}
}
func importKeyFor(access *filesystemAccess, canonical string, synthetic bool) importNodeKey {
kind := "host"
resourceKind := "file"
resourcePath := filepath.ToSlash(canonical)
if access.native != nil {
kind = "native"
}
if synthetic {
resourceKind = "synthetic"
resourcePath = ""
}
return importNodeKey{
namespaceKind: kind,
// This traversal owns the main input position; grouped inputs receive
// their explicit positions when their acquisition graphs are combined.
namespaceAnchor: fmt.Sprintf("%020d/%020d", 0, 0),
resourceKind: resourceKind, resourcePath: resourcePath,
}
}
func importKeyForMainSource(source *SourceInfo) importNodeKey {
if source == nil {
return importNodeKey{}
}
namespaceKind := "host"
if source.Kind == SourceKindFilesystem {
namespaceKind = "native"
}
synthetic := source.Kind != SourceKindFile && source.Kind != SourceKindFilesystem
resourceKind := "file"
resourcePath := filepath.ToSlash(source.Path)
if synthetic {
resourceKind = "synthetic"
resourcePath = ""
}
return importNodeKey{
namespaceKind: namespaceKind, namespaceAnchor: fmt.Sprintf("%020d/%020d", 0, 0),
resourceKind: resourceKind, resourcePath: resourcePath,
}
}

View File

@@ -0,0 +1,387 @@
package engine
import (
"errors"
"io/fs"
"path"
"path/filepath"
"sort"
"strings"
)
type importSelectionOutcome uint8
const (
importSelectionNone importSelectionOutcome = iota
importSelectionHasCandidates
)
type selectedImportCandidate struct {
path string
accessPath string
label string
site importFieldSite
}
// importSelection contains lexical candidates only; admission assigns source identity later.
type importSelection struct {
candidates []selectedImportCandidate
outcome importSelectionOutcome
}
func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy) (importSelection, []Diagnostic) {
normalized := make([]selectedImportCandidate, 0, len(spec.paths))
seen := make(map[string]struct{}, len(spec.paths))
for index, relative := range spec.paths {
accessPath := access.join(basePath, relative)
candidatePath := accessPath
if access.native == nil {
candidatePath = filepath.ToSlash(filepath.Clean(accessPath))
}
if _, exists := seen[candidatePath]; exists {
continue
}
seen[candidatePath] = struct{}{}
var site importFieldSite
if index < len(spec.selectorSites) {
site = spec.selectorSites[index]
} else {
site = spec.fieldSites["path"]
}
normalized = append(normalized, selectedImportCandidate{path: candidatePath, accessPath: accessPath, label: relativeImportLabel(access, basePath, accessPath), site: site})
}
sort.Slice(normalized, func(left, right int) bool {
return normalized[left].path < normalized[right].path
})
candidates := make([]selectedImportCandidate, 0, len(normalized))
for _, candidate := range normalized {
missing, _, diagnostic := validateNamedImportTarget(access, candidate, policy.required, false)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if missing {
continue
}
if excludedBasename(path.Base(candidate.path), policy.exclude) {
continue
}
candidates = append(candidates, candidate)
}
selection := importSelection{candidates: candidates}
if len(candidates) != 0 {
selection.outcome = importSelectionHasCandidates
}
return selection, nil
}
func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
roots := make([]selectedImportCandidate, 0, len(spec.paths))
seenRoots := make(map[string]struct{}, len(spec.paths))
for index, relative := range spec.paths {
accessPath := access.join(basePath, relative)
candidatePath := normalizedImportCandidatePath(access, accessPath)
if _, exists := seenRoots[candidatePath]; exists {
continue
}
seenRoots[candidatePath] = struct{}{}
site := spec.fieldSites["dir"]
if index < len(spec.selectorSites) {
site = spec.selectorSites[index]
}
roots = append(roots, selectedImportCandidate{path: candidatePath, accessPath: accessPath, site: site})
}
sort.Slice(roots, func(left, right int) bool { return roots[left].path < roots[right].path })
if accounting == nil {
accounting = &importAccounting{}
}
discovered := make([]selectedImportCandidate, 0)
seenCandidates := make(map[string]struct{})
// Each root is traversed once. Filename patterns are tested together for
// every discovered file, and seenCandidates forms their set union across
// overlapping roots. Exclusions are applied once below as set difference.
for _, root := range roots {
missing, isDirectory, diagnostic := validateNamedImportTarget(access, root, false, true)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if missing {
continue
}
if !isDirectory {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")}
}
rootTarget, targetErr := access.importEntryTarget(root.accessPath, nil, true)
if targetErr != nil {
return importSelection{}, []Diagnostic{*importDirectoryTargetDiagnostic(root.site, targetErr)}
}
if access.native == nil {
root.accessPath = rootTarget.identity
}
entries, missing, diagnostic := importDirectoryEntries(access, root.accessPath, root.site, limits, accounting)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if missing {
continue
}
stack := []importDirectoryFrame{{directory: root, depth: 0, entries: entries, identity: rootTarget.identity}}
// Active-path cycle detection removes each directory when its frame is
// popped, so a later non-active alias remains a separate traversal.
activeDirectories := map[string]struct{}{rootTarget.identity: {}}
for len(stack) != 0 {
frameIndex := len(stack) - 1
frame := &stack[frameIndex]
if frame.next >= len(frame.entries) {
delete(activeDirectories, frame.identity)
stack = stack[:frameIndex]
continue
}
entry := frame.entries[frame.next]
frame.next++
accessPath := access.join(frame.directory.accessPath, entry.Name())
lexicalPath := access.join(frame.directory.path, entry.Name())
candidate := selectedImportCandidate{
path: normalizedImportCandidatePath(access, lexicalPath), accessPath: accessPath,
label: relativeImportLabel(access, basePath, lexicalPath), site: frame.directory.site,
}
if !policy.hidden && strings.HasPrefix(entry.Name(), ".") {
continue
}
target, entryErr := access.importEntryTarget(candidate.accessPath, entry, policy.follow)
if entryErr != nil {
if errors.Is(entryErr, fs.ErrNotExist) {
continue
}
return importSelection{}, []Diagnostic{*importDirectoryTargetDiagnostic(frame.directory.site, entryErr)}
}
if target.symlink && !policy.follow {
continue
}
if target.dir {
if !directoryDepthAllows(policy, frame.depth) {
continue
}
if _, exists := activeDirectories[target.identity]; exists {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(frame.directory.site, CodeImportCycle, "directory traversal cycle detected")}
}
childMissing, childIsDirectory, childDiagnostic := validateNamedImportTarget(access, candidate, false, true)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if childMissing || !childIsDirectory {
continue
}
childEntries, vanished, childDiagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if !vanished {
if access.native == nil {
candidate.accessPath = target.identity
}
activeDirectories[target.identity] = struct{}{}
stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries, identity: target.identity})
}
continue
}
if target.identity != "" && access.native == nil {
candidate.accessPath = target.identity
} else {
canonical, canonicalErr := access.canonical(accessPath)
if canonicalErr != nil {
if errors.Is(canonicalErr, fs.ErrNotExist) {
continue
}
return importSelection{}, []Diagnostic{globFilesystemDiagnostic(canonicalErr, "directory search file could not be canonicalized", frame.directory.site.location)}
}
candidate.accessPath = canonical
}
if !matchesImportFilename(entry.Name(), policy.pattern) {
continue
}
if _, exists := seenCandidates[candidate.path]; exists {
continue
}
seenCandidates[candidate.path] = struct{}{}
discovered = append(discovered, candidate)
}
}
sort.Slice(discovered, func(left, right int) bool { return discovered[left].path < discovered[right].path })
selected := make([]selectedImportCandidate, 0, len(discovered))
for _, candidate := range discovered {
missing, isDirectory, diagnostic := validateNamedImportTarget(access, candidate, false, true)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if missing || isDirectory || excludedBasename(path.Base(candidate.path), policy.exclude) {
continue
}
selected = append(selected, candidate)
}
if policy.required && len(selected) == 0 {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(spec.fieldSites["dir"], CodeImportMissing, "required directory import selected no files")}
}
selection := importSelection{candidates: selected}
if len(selected) != 0 {
selection.outcome = importSelectionHasCandidates
}
return selection, nil
}
type importDirectoryFrame struct {
directory selectedImportCandidate
depth int
entries []fs.DirEntry
next int
identity string
}
func importDirectoryTargetDiagnostic(site importFieldSite, err error) *Diagnostic {
if isImportLinkCycle(err) {
return importSelectionDiagnostic(site, CodeImportCycle, "directory traversal cycle detected")
}
if errors.Is(err, errImportPathOutsideRoot) {
return importSelectionDiagnostic(site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
}
return importSelectionDiagnostic(site, CodeImportInvalid, "directory entry could not be inspected")
}
func importDirectoryEntries(access *filesystemAccess, directory string, site importFieldSite, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, *Diagnostic) {
target, targetErr := access.importEntryTarget(directory, nil, true)
if targetErr != nil {
if errors.Is(targetErr, fs.ErrNotExist) {
return nil, true, nil
}
return nil, false, importDirectoryTargetDiagnostic(site, targetErr)
}
readPath := target.identity
if readPath == "" {
readPath = directory
}
entries := make([]fs.DirEntry, 0)
err := visitBoundedImportDirectory(access, readPath, limits.entries, accounting, func(entry fs.DirEntry) error {
entries = append(entries, entry)
return nil
})
if errors.Is(err, errImportDirectoryEntryLimit) {
return nil, false, importSelectionDiagnostic(site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
}
if errors.Is(err, fs.ErrNotExist) {
return nil, true, nil
}
if err != nil {
return nil, false, importSelectionDiagnostic(site, CodeImportInvalid, "directory search root could not be read")
}
sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
return entries, false, nil
}
func directoryDepthAllows(policy resolvedImportPolicy, currentDepth int) bool {
if !policy.recursive {
return false
}
childDepth := currentDepth + 1
return !policy.depthSet || policy.depth == 0 || childDepth < policy.depth
}
func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string {
if access.native != nil {
return accessPath
}
return filepath.ToSlash(filepath.Clean(accessPath))
}
func relativeImportLabel(access *filesystemAccess, basePath, candidatePath string) string {
if access.native != nil {
if candidatePath == basePath {
return "."
}
prefix := strings.TrimSuffix(basePath, "/") + "/"
if basePath == "." {
return candidatePath
}
if strings.HasPrefix(candidatePath, prefix) {
return strings.TrimPrefix(candidatePath, prefix)
}
return candidatePath
}
label, err := filepath.Rel(basePath, candidatePath)
if err == nil {
return filepath.ToSlash(label)
}
return filepath.ToSlash(candidatePath)
}
func matchesImportFilename(name string, patterns []string) bool {
if patterns == nil {
return true
}
for _, pattern := range patterns {
matched, err := path.Match(pattern, name)
if err == nil && matched {
return true
}
}
return false
}
func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required, allowDirectory bool) (bool, bool, *Diagnostic) {
if !access.contains(candidate.accessPath) {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root")
}
canonical, err := access.canonical(candidate.accessPath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
if required {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
}
return true, false, nil
}
if errors.Is(err, fs.ErrPermission) {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
}
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized")
}
if !access.contains(canonical) {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
}
info, err := access.stat(canonical)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
if required {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
}
return true, false, nil
}
if errors.Is(err, fs.ErrPermission) {
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
}
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected")
}
if info.IsDir() && allowDirectory {
return false, true, nil
}
if !info.Mode().IsRegular() {
return false, false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file")
}
return false, false, nil
}
func excludedBasename(name string, patterns []string) bool {
for _, pattern := range patterns {
matched, err := path.Match(pattern, name)
if err == nil && matched {
return true
}
}
return false
}
func importSelectionDiagnostic(site importFieldSite, code Code, message string) *Diagnostic {
location := site.location
return &Diagnostic{
Code: code, Severity: SeverityError, Path: site.path.clone(),
Message: message, Primary: &location,
}
}

View File

@@ -0,0 +1,257 @@
package engine
import (
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestSelectNamedImportsHandlesPartialAndEmptySelections(t *testing.T) {
t.Parallel()
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "present.yaml"), []byte("value: present\n"), 0o600); err != nil {
t.Fatal(err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - path: [missing.yaml, present.yaml]\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectNamedImportCandidates(access, root, specs[0], resolvedImportPolicy{})
if len(diagnostics) != 0 {
t.Fatalf("optional partial selection diagnostics = %#v", diagnostics)
}
if selection.outcome != importSelectionHasCandidates || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "present.yaml" {
t.Fatalf("partial selection = %#v", selection)
}
requiredSpecs, diagnostics := parseImportSpecsYAML(t, " - path: [missing.yaml, present.yaml]\n required: true\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectNamedImportCandidates(access, root, requiredSpecs[0], resolvedImportPolicy{required: true})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
t.Fatalf("required partial selection diagnostics = %#v", diagnostics)
}
emptySpecs, diagnostics := parseImportSpecsYAML(t, " - path: []\n required: true\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
empty, diagnostics := selectNamedImportCandidates(access, root, emptySpecs[0], resolvedImportPolicy{required: true})
if len(diagnostics) != 0 || empty.outcome != importSelectionNone || len(empty.candidates) != 0 {
t.Fatalf("required empty path selection, diagnostics = %#v, %#v", empty, diagnostics)
}
}
func TestSelectNamedImportsChecksRequiredTargetsBeforeExclusions(t *testing.T) {
t.Parallel()
root := t.TempDir()
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - path: missing.yaml\n required: true\n exclude: '*.yaml'\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectNamedImportCandidates(access, root, specs[0], policy)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing || diagnostics[0].Path.String() != "/configkit/imports/0/path" {
t.Fatalf("required excluded missing target diagnostics = %#v", diagnostics)
}
}
func TestSelectNamedImportsDeduplicatesLexicalPathsButRetainsAliases(t *testing.T) {
t.Parallel()
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "real.yaml"), []byte("value: real\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink("real.yaml", filepath.Join(root, "alias.yaml")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - path: [real.yaml, nested/../real.yaml, alias.yaml]\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectNamedImportCandidates(access, root, specs[0], resolvedImportPolicy{})
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
if len(selection.candidates) != 2 {
t.Fatalf("candidates = %#v, want two alias spellings", selection.candidates)
}
if filepath.Base(selection.candidates[0].path) != "alias.yaml" || filepath.Base(selection.candidates[1].path) != "real.yaml" {
t.Fatalf("candidate order = %#v", selection.candidates)
}
}
func TestSelectNamedImportsDoesNotOpenExcludedContentOrApplySearchControls(t *testing.T) {
t.Parallel()
filesystem := &statOnlyImportFilesystem{files: map[string]bool{"secret.yaml": true, ".private.yaml": true}}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, " - path: secret.yaml\n required: true\n exclude: '*.yaml'\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectNamedImportCandidates(access, ".", specs[0], policy)
if len(diagnostics) != 0 || selection.outcome != importSelectionNone {
t.Fatalf("excluded selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if filesystem.openCalls != 0 || filesystem.statCalls == 0 {
t.Fatalf("excluded target access = %d opens, %d stats", filesystem.openCalls, filesystem.statCalls)
}
private, diagnostics := parseImportSpecsYAML(t, " - path: .private.yaml\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
hidden := false
privatePolicy, policyDiagnostics := resolveImportPolicy(private[0], importDefaults{hidden: &hidden})
if len(policyDiagnostics) != 0 || !privatePolicy.hidden {
t.Fatalf("path selector accepted search-only defaults: %#v, %#v", privatePolicy, policyDiagnostics)
}
selection, diagnostics = selectNamedImportCandidates(access, ".", private[0], privatePolicy)
if len(diagnostics) != 0 || len(selection.candidates) != 1 || selection.candidates[0].path != ".private.yaml" {
t.Fatalf("explicit hidden path was filtered: %#v, %#v", selection, diagnostics)
}
if filesystem.openCalls != 0 {
t.Fatalf("named selection opened content %d times", filesystem.openCalls)
}
}
func TestSelectNamedImportsChecksNormalizedPathsLexically(t *testing.T) {
t.Parallel()
filesystem := &statOnlyImportFilesystem{files: map[string]bool{"a.yaml": true, "z.yaml": true}}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, " - path: [z.yaml, a.yaml]\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectNamedImportCandidates(access, ".", specs[0], resolvedImportPolicy{})
if len(diagnostics) != 0 || len(selection.candidates) != 2 {
t.Fatalf("selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if strings.Join(filesystem.statPaths, ",") != "a.yaml,a.yaml,z.yaml,z.yaml" {
t.Fatalf("target stat order = %#v", filesystem.statPaths)
}
}
func TestSelectNamedImportsDoesNotTreatInvalidTargetsAsAbsence(t *testing.T) {
t.Parallel()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "directory"), 0o700); err != nil {
t.Fatal(err)
}
outside := filepath.Join(t.TempDir(), "outside.yaml")
if err := os.WriteFile(outside, []byte("value: outside\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(root, "escape.yaml")); err != nil {
t.Skipf("symlinks are unavailable: %v", err)
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
for _, test := range []struct {
name, value string
code Code
}{
{name: "nonregular target", value: "directory", code: CodeSourceNotRegular},
{name: "outside symlink", value: "escape.yaml", code: CodeImportOutsideRoot},
} {
t.Run(test.name, func(t *testing.T) {
specs, diagnostics := parseImportSpecsYAML(t, " - path: "+test.value+"\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectNamedImportCandidates(access, root, specs[0], resolvedImportPolicy{})
if len(diagnostics) != 1 || diagnostics[0].Code != test.code {
t.Fatalf("diagnostics = %#v, want %s", diagnostics, test.code)
}
})
}
}
func TestSelectNamedImportsDoesNotTreatPermissionErrorsAsAbsence(t *testing.T) {
t.Parallel()
filesystem := &statOnlyImportFilesystem{statErrors: map[string]error{"denied.yaml": fs.ErrPermission}}
access := newNativeAccess(filesystem, ".", &namespaceToken{})
specs, diagnostics := parseImportSpecsYAML(t, " - path: denied.yaml\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectNamedImportCandidates(access, ".", specs[0], resolvedImportPolicy{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
t.Fatalf("optional permission failure diagnostics = %#v", diagnostics)
}
}
type statOnlyImportFilesystem struct {
files map[string]bool
statErrors map[string]error
openCalls int
statCalls int
statPaths []string
}
func (filesystem *statOnlyImportFilesystem) Open(name string) (fs.File, error) {
filesystem.openCalls++
return nil, fs.ErrPermission
}
func (filesystem *statOnlyImportFilesystem) Stat(name string) (fs.FileInfo, error) {
filesystem.statCalls++
filesystem.statPaths = append(filesystem.statPaths, name)
if err := filesystem.statErrors[name]; err != nil {
return nil, err
}
if !filesystem.files[name] {
return nil, fs.ErrNotExist
}
return importSelectionFileInfo{name: pathBaseForTest(name)}, nil
}
type importSelectionFileInfo struct{ name string }
func (info importSelectionFileInfo) Name() string { return info.name }
func (importSelectionFileInfo) Size() int64 { return 1 }
func (importSelectionFileInfo) Mode() fs.FileMode { return 0o600 }
func (importSelectionFileInfo) ModTime() time.Time { return time.Time{} }
func (importSelectionFileInfo) IsDir() bool { return false }
func (importSelectionFileInfo) Sys() any { return nil }
func pathBaseForTest(name string) string { return strings.TrimPrefix(name, "./") }

View File

@@ -0,0 +1,471 @@
package engine
import (
"path"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
type importSelector uint8
const (
importSelectorPath importSelector = iota + 1
importSelectorDirectory
importSelectorGlob
)
type importFieldSite struct {
path Path
location Location
}
// importDeclarationSpec preserves authored presence until discovery policy is resolved.
type importDeclarationSpec struct {
path Path
location Location
selector importSelector
paths []string
selectorSites []importFieldSite
glob string
globSegments []string
required *bool
recursive *bool
follow *bool
hidden *bool
pattern []string
patternSet bool
exclude []string
excludeSet bool
depth int
depthSet bool
destination Path
placement placementMode
fieldSites map[string]importFieldSite
}
// importDefaults are the engine's private copy of load-wide discovery defaults.
type importDefaults struct {
required *bool
follow *bool
hidden *bool
recursive *bool
pattern []string
}
// ImportDefaults transports root-owned load defaults across the facade
// boundary. Load options snapshot this value before source acquisition.
type ImportDefaults struct {
// Required controls absence handling when non-nil.
Required *bool
// Follow controls symlink following when non-nil.
Follow *bool
// Hidden controls hidden-entry discovery when non-nil.
Hidden *bool
// Recursive controls descendant directory traversal when non-nil.
Recursive *bool
// Pattern filters directory-search filenames; nil and empty differ.
Pattern []string
}
// OptionsWithImportDefaults copies root-owned defaults into ordinary options.
func OptionsWithImportDefaults[T any](options Options[T], value ImportDefaults) Options[T] {
options.importDefaults = privateImportDefaults(value)
return options
}
// LayeredOptionsWithImportDefaults copies root-owned defaults into grouped options.
func LayeredOptionsWithImportDefaults[T any](options LayeredOptions[T], value ImportDefaults) LayeredOptions[T] {
options.importDefaults = privateImportDefaults(value)
return options
}
func privateImportDefaults(value ImportDefaults) importDefaults {
return snapshotImportDefaults(importDefaults{
required: value.Required, follow: value.Follow, hidden: value.Hidden,
recursive: value.Recursive, pattern: value.Pattern,
})
}
type resolvedImportPolicy struct {
required bool
follow bool
hidden bool
recursive bool
pattern []string
depth int
depthSet bool
exclude []string
destination Path
placement placementMode
}
func snapshotImportDefaults(value importDefaults) importDefaults {
return importDefaults{
required: cloneBool(value.required),
follow: cloneBool(value.follow),
hidden: cloneBool(value.hidden),
recursive: cloneBool(value.recursive),
pattern: cloneStrings(value.pattern),
}
}
func cloneBool(value *bool) *bool {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneStrings(values []string) []string {
if values == nil {
return nil
}
return append(make([]string, 0, len(values)), values...)
}
func validateImportDefaults(value importDefaults) []Diagnostic {
for _, pattern := range value.pattern {
if _, err := path.Match(pattern, ""); err != nil {
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "import default pattern is invalid")}
}
}
return nil
}
func resolveImportPolicy(spec importDeclarationSpec, defaults importDefaults) (resolvedImportPolicy, []Diagnostic) {
defaults = snapshotImportDefaults(defaults)
policy := resolvedImportPolicy{
required: boolOr(spec.required, defaults.required, false),
follow: true,
hidden: true,
recursive: false,
destination: spec.destination.clone(),
placement: spec.placement,
depth: spec.depth,
depthSet: spec.depthSet,
exclude: cloneStrings(spec.exclude),
}
if spec.selector == importSelectorDirectory || spec.selector == importSelectorGlob {
policy.follow = boolOr(spec.follow, defaults.follow, true)
policy.hidden = boolOr(spec.hidden, defaults.hidden, true)
}
if spec.selector == importSelectorDirectory {
policy.recursive = boolOr(spec.recursive, defaults.recursive, false)
if spec.patternSet {
policy.pattern = cloneStrings(spec.pattern)
} else {
policy.pattern = cloneStrings(defaults.pattern)
}
}
if spec.depthSet && !policy.recursive {
site := spec.fieldSites["depth"]
location := site.location
return resolvedImportPolicy{}, []Diagnostic{{
Code: CodeImportInvalid, Severity: SeverityError,
Path: site.path.clone(), Message: "depth requires effective recursive discovery",
Primary: &location,
}}
}
return policy, nil
}
func boolOr(explicit, application *bool, fallback bool) bool {
if explicit != nil {
return *explicit
}
if application != nil {
return *application
}
return fallback
}
// parseImportSpecificationDirective validates every declaration before callers select candidates.
func parseImportSpecificationDirective(node *document.Node, directivePath Path) ([]importDeclarationSpec, []Diagnostic) {
if node.Kind != document.Mapping {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)}
}
var specs []importDeclarationSpec
var diagnostics []Diagnostic
for _, entry := range node.Mapping {
fieldPath := directivePath.Key(entry.Key)
if entry.Key != "imports" {
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value))
continue
}
parsed, issues := parseImportDeclarations(entry.Value, fieldPath)
specs = append(specs, parsed...)
diagnostics = append(diagnostics, issues...)
}
return specs, canonicalDiagnostics(diagnostics)
}
func parseImportDeclarations(node *document.Node, declarationPath Path) ([]importDeclarationSpec, []Diagnostic) {
if node == nil || node.Kind != document.Sequence {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "imports must be a sequence", node)}
}
var specs []importDeclarationSpec
var diagnostics []Diagnostic
for index, declarationNode := range node.Sequence {
spec, issues := parseImportDeclarationSpec(declarationNode, declarationPath.Index(index))
if len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
continue
}
specs = append(specs, spec)
}
return specs, canonicalDiagnostics(diagnostics)
}
func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (importDeclarationSpec, []Diagnostic) {
if node.Kind != document.Mapping {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)}
}
spec := importDeclarationSpec{
path: declarationPath.clone(),
location: publicLocation(node.Location),
placement: placementModeMapping,
fieldSites: make(map[string]importFieldSite, len(node.Mapping)),
}
selectors := 0
selectorIndex := -1
for index, entry := range node.Mapping {
if selectorForField(entry.Key) != 0 {
selectors++
selectorIndex = index
}
}
if selectors != 1 {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration requires exactly one path, dir, or glob selector", node)}
}
selectorEntry := node.Mapping[selectorIndex]
spec.selector = selectorForField(selectorEntry.Key)
for _, entry := range node.Mapping {
fieldPath := declarationPath.Key(entry.Key)
spec.fieldSites[entry.Key] = importFieldSite{path: fieldPath.clone(), location: publicLocation(entry.Value.Location)}
if selectorForField(entry.Key) != 0 {
continue
}
if !importFieldApplies(entry.Key, spec.selector) {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not supported by this selector", entry.Value)}
}
switch entry.Key {
case "required", "recursive", "follow", "hidden":
if entry.Value.Kind != document.Boolean {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, entry.Key+" must be a boolean", entry.Value)}
}
value := entry.Value.Text == "true"
switch entry.Key {
case "required":
spec.required = &value
case "recursive":
spec.recursive = &value
case "follow":
spec.follow = &value
case "hidden":
spec.hidden = &value
}
case "pattern":
values, origins, ok, badIndex := parseStringSet(entry.Value)
if !ok {
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(entry.Value, fieldPath, "pattern", badIndex)}
}
if badIndex, ok := invalidWildcard(values); ok {
badPath, badNode := listMemberSite(entry.Value, fieldPath, origins[badIndex])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, "pattern contains an invalid wildcard expression", badNode)}
}
spec.pattern, spec.patternSet = values, true
case "exclude":
values, origins, ok, badIndex := parseStringSet(entry.Value)
if !ok {
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(entry.Value, fieldPath, "exclude", badIndex)}
}
if badIndex, ok := invalidWildcard(values); ok {
badPath, badNode := listMemberSite(entry.Value, fieldPath, origins[badIndex])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, "exclude contains an invalid wildcard expression", badNode)}
}
spec.exclude, spec.excludeSet = values, true
case "depth":
depth, ok := parseImportDepth(entry.Value)
if !ok {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "depth must be a nonnegative integer", entry.Value)}
}
spec.depth, spec.depthSet = depth, true
case "destination":
destination, diagnostic := parseImportDestination(entry.Value, fieldPath)
if diagnostic != nil {
return importDeclarationSpec{}, []Diagnostic{*diagnostic}
}
spec.destination = destination
case "placement":
if entry.Value.Kind != document.String {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
switch entry.Value.Text {
case "mapping":
spec.placement = placementModeMapping
case "sequence-item":
spec.placement = placementModeSequenceItem
default:
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
default:
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)}
}
}
selectorPath := declarationPath.Key(selectorEntry.Key)
spec.fieldSites[selectorEntry.Key] = importFieldSite{path: selectorPath.clone(), location: publicLocation(selectorEntry.Value.Location)}
switch spec.selector {
case importSelectorPath, importSelectorDirectory:
values, origins, ok, badIndex := parseStringSet(selectorEntry.Value)
if !ok {
name := selectorEntry.Key
return importDeclarationSpec{}, []Diagnostic{stringSetDiagnostic(selectorEntry.Value, selectorPath, name, badIndex)}
}
for index, value := range values {
if !validImportPath(value) {
badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, selectorEntry.Key+" contains an invalid source path", badNode)}
}
badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
spec.selectorSites = append(spec.selectorSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
}
spec.paths = values
case importSelectorGlob:
if selectorEntry.Value.Kind != document.String || !validImportPath(selectorEntry.Value.Text) {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob must be a valid nonempty path pattern string", selectorEntry.Value)}
}
segments, ok := validGlobPattern(selectorEntry.Value.Text)
if !ok {
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, selectorPath, "glob pattern is invalid", selectorEntry.Value)}
}
spec.glob, spec.globSegments = selectorEntry.Value.Text, segments
}
return spec, nil
}
func selectorForField(field string) importSelector {
switch field {
case "path":
return importSelectorPath
case "dir":
return importSelectorDirectory
case "glob":
return importSelectorGlob
default:
return 0
}
}
func importFieldApplies(field string, selector importSelector) bool {
switch field {
case "required", "exclude", "destination", "placement":
return true
case "pattern", "recursive", "depth":
return selector == importSelectorDirectory
case "follow", "hidden":
return selector == importSelectorDirectory || selector == importSelectorGlob
default:
return false
}
}
func parseStringSet(node *document.Node) ([]string, []int, bool, int) {
if node.Kind == document.String {
return []string{node.Text}, []int{-1}, true, -1
}
if node.Kind != document.Sequence {
return nil, nil, false, -1
}
values := make([]string, 0, len(node.Sequence))
origins := make([]int, 0, len(node.Sequence))
seen := make(map[string]struct{}, len(node.Sequence))
for index, item := range node.Sequence {
if item.Kind != document.String {
return nil, nil, false, index
}
if _, exists := seen[item.Text]; exists {
continue
}
seen[item.Text] = struct{}{}
values = append(values, item.Text)
origins = append(origins, index)
}
return values, origins, true, -1
}
func stringSetDiagnostic(node *document.Node, fieldPath Path, field string, badIndex int) Diagnostic {
if badIndex >= 0 {
badPath, item := listMemberSite(node, fieldPath, badIndex)
return nodeImportDiagnostic(CodeImportInvalid, badPath, field+" members must be strings", item)
}
return nodeImportDiagnostic(CodeImportInvalid, fieldPath, field+" must be a string or a flat list of strings", node)
}
func invalidWildcard(values []string) (int, bool) {
for index, value := range values {
if _, err := path.Match(value, ""); err != nil {
return index, true
}
}
return -1, false
}
func listMemberSite(node *document.Node, fieldPath Path, index int) (Path, *document.Node) {
if node.Kind != document.Sequence || index < 0 {
return fieldPath, node
}
return fieldPath.Index(index), node.Sequence[index]
}
func parseImportDepth(node *document.Node) (int, bool) {
if node.Kind != document.Number || node.Text == "" {
return 0, false
}
if node.Text == "-0" {
return 0, true
}
var value uint64
maxInt := uint64(^uint(0) >> 1)
for index := 0; index < len(node.Text); index++ {
character := node.Text[index]
if character < '0' || character > '9' {
return 0, false
}
digit := uint64(character - '0')
if value > (maxInt-digit)/10 {
value = maxInt
continue // Saturate the bound, but validate every remaining character.
}
value = value*10 + digit
}
return int(value), true
}
func parseImportDestination(node *document.Node, fieldPath Path) (Path, *Diagnostic) {
if node.Kind != document.Sequence {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must be a sequence of strings", node)
return Path{}, &diagnostic
}
relative := Path{segments: make([]pathSegment, 0, len(node.Sequence))}
for index, item := range node.Sequence {
memberPath := fieldPath.Index(index)
if item.Kind != document.String {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be strings", item)
return Path{}, &diagnostic
}
member := Path{segments: []pathSegment{{kind: pathSegmentKey, key: item.Text}}}
if _, err := normalizeDestination(member); err != nil {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, memberPath, "destination members must be non-reserved mapping keys", item)
return Path{}, &diagnostic
}
relative.segments = append(relative.segments, pathSegment{kind: pathSegmentKey, key: item.Text})
}
normalized, err := normalizeDestination(relative)
if err != nil {
diagnostic := nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", node)
return Path{}, &diagnostic
}
return normalized, nil
}

View File

@@ -0,0 +1,243 @@
package engine
import (
"errors"
"io"
"strings"
"testing"
)
func TestParseImportDeclarationSpecNormalizesPresenceAndPlacement(t *testing.T) {
t.Parallel()
specs, diagnostics := parseImportSpecsYAML(t, `
- path: [one.yaml, one.yaml, two.yaml]
required: false
destination: [services]
placement: sequence-item
- dir: []
recursive: true
pattern: []
exclude: []
- glob: config/*.yaml
`)
if len(diagnostics) != 0 {
t.Fatalf("diagnostics = %#v", diagnostics)
}
if len(specs) != 3 {
t.Fatalf("spec count = %d, want 3", len(specs))
}
first := specs[0]
if first.selector != importSelectorPath || len(first.paths) != 2 || first.paths[0] != "one.yaml" || first.paths[1] != "two.yaml" {
t.Fatalf("path selector = %#v", first)
}
if first.required == nil || *first.required || first.destination.String() != "/services" || first.placement != placementModeSequenceItem {
t.Fatalf("explicit declaration values = %#v", first)
}
if first.path.String() != "/configkit/imports/0" || first.fieldSites["required"].path.String() != "/configkit/imports/0/required" {
t.Fatalf("stored paths = declaration %q, required %q", first.path.String(), first.fieldSites["required"].path.String())
}
second := specs[1]
if second.selector != importSelectorDirectory || second.paths == nil || len(second.paths) != 0 || !second.patternSet || second.pattern == nil || len(second.pattern) != 0 || !second.excludeSet || second.exclude == nil {
t.Fatalf("empty list presence was lost: %#v", second)
}
third := specs[2]
if third.selector != importSelectorGlob || third.glob != "config/*.yaml" || len(third.globSegments) != 2 {
t.Fatalf("glob selector = %#v", third)
}
}
func TestParseImportDeclarationSpecValidatesSelectorMatrixAndTypes(t *testing.T) {
t.Parallel()
tests := []struct {
name, yaml, path string
}{
{name: "missing selector", yaml: " - required: true\n", path: "/configkit/imports/0"},
{name: "multiple selectors", yaml: " - path: one.yaml\n dir: fragments\n", path: "/configkit/imports/0"},
{name: "removed optional", yaml: " - path: one.yaml\n optional: true\n", path: "/configkit/imports/0/optional"},
{name: "inapplicable field", yaml: " - path: one.yaml\n pattern: '*.yaml'\n", path: "/configkit/imports/0/pattern"},
{name: "selector null", yaml: " - path: null\n", path: "/configkit/imports/0/path"},
{name: "empty path", yaml: " - path: ''\n", path: "/configkit/imports/0/path"},
{name: "nested selector list", yaml: " - path: [[one.yaml]]\n", path: "/configkit/imports/0/path/0"},
{name: "null selector member", yaml: " - dir: [fragments, null]\n", path: "/configkit/imports/0/dir/1"},
{name: "empty glob", yaml: " - glob: ''\n", path: "/configkit/imports/0/glob"},
{name: "recursive glob wildcard", yaml: " - glob: '**/*.yaml'\n", path: "/configkit/imports/0/glob"},
{name: "null boolean", yaml: " - path: one.yaml\n required: null\n", path: "/configkit/imports/0/required"},
{name: "bad pattern member with empty roots", yaml: " - dir: []\n pattern: ['*', '[']\n", path: "/configkit/imports/0/pattern/1"},
{name: "bad exclusion member", yaml: " - glob: '*.yaml'\n exclude: ['[']\n", path: "/configkit/imports/0/exclude/0"},
{name: "invalid destination", yaml: " - path: one.yaml\n destination: [configkit]\n", path: "/configkit/imports/0/destination/0"},
{name: "invalid placement", yaml: " - path: one.yaml\n placement: sequence\n", path: "/configkit/imports/0/placement"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
_, diagnostics := parseImportSpecsYAML(t, test.yaml)
if len(diagnostics) == 0 {
t.Fatal("expected a declaration diagnostic")
}
if diagnostics[0].Code != CodeImportInvalid || diagnostics[0].Path.String() != test.path {
t.Fatalf("diagnostic = %#v, want %s at %s", diagnostics[0], CodeImportInvalid, test.path)
}
if diagnostics[0].Primary == nil || diagnostics[0].Primary.Line == 0 {
t.Fatalf("diagnostic has no source location: %#v", diagnostics[0])
}
})
}
}
func TestParseImportDeclarationSpecValidatesDepthSyntaxAndPolicy(t *testing.T) {
t.Parallel()
for _, test := range []struct {
value string
valid bool
want int
}{
{value: "0", valid: true, want: 0},
{value: "-0", valid: true, want: 0},
{value: "999999999999999999999999999999999999", valid: true, want: int(^uint(0) >> 1)},
{value: "-1"},
{value: "1.5"},
{value: "1e3"},
{value: "'2'"},
{value: "true"},
{value: "null"},
} {
t.Run(test.value, func(t *testing.T) {
t.Parallel()
specs, diagnostics := parseImportSpecsYAML(t, " - dir: []\n recursive: true\n depth: "+test.value+"\n")
if !test.valid {
if len(diagnostics) == 0 || diagnostics[0].Path.String() != "/configkit/imports/0/depth" {
t.Fatalf("diagnostics = %#v, want invalid depth", diagnostics)
}
return
}
if len(diagnostics) != 0 || len(specs) != 1 || specs[0].depth != test.want {
t.Fatalf("specs, diagnostics = %#v, %#v; want depth %d", specs, diagnostics, test.want)
}
})
}
specs, diagnostics := parseImportSpecsYAML(t, " - dir: []\n depth: 1\n")
if len(diagnostics) != 0 {
t.Fatalf("syntax-only parse diagnostics = %#v", diagnostics)
}
_, diagnostics = resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 1 || diagnostics[0].Path.String() != "/configkit/imports/0/depth" {
t.Fatalf("policy diagnostics = %#v, want depth applicability error", diagnostics)
}
recursive := true
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{recursive: &recursive})
if len(diagnostics) != 0 || !policy.recursive || policy.depth != 1 || !policy.depthSet {
t.Fatalf("application-default policy, diagnostics = %#v, %#v", policy, diagnostics)
}
}
func TestResolveImportPolicyAppliesDefaultsIndependently(t *testing.T) {
t.Parallel()
omittedSpecs, diagnostics := parseImportSpecsYAML(t, " - dir: fragments\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
omitted, diagnostics := resolveImportPolicy(omittedSpecs[0], importDefaults{})
if len(diagnostics) != 0 || omitted.required || omitted.recursive || !omitted.follow || !omitted.hidden || omitted.pattern != nil {
t.Fatalf("library defaults = %#v, diagnostics = %#v", omitted, diagnostics)
}
specs, diagnostics := parseImportSpecsYAML(t, `
- dir: fragments
required: false
recursive: false
follow: false
hidden: false
pattern: []
`)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
appRequired, appRecursive, appFollow, appHidden := true, true, true, true
defaults := importDefaults{
required: &appRequired,
recursive: &appRecursive,
follow: &appFollow,
hidden: &appHidden,
pattern: []string{"*.yaml"},
}
policy, diagnostics := resolveImportPolicy(specs[0], defaults)
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
if policy.required || policy.recursive || policy.follow || policy.hidden || policy.pattern == nil || len(policy.pattern) != 0 {
t.Fatalf("explicit values did not override application defaults: %#v", policy)
}
pathSpecs, diagnostics := parseImportSpecsYAML(t, " - path: file.yaml\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
pathPolicy, diagnostics := resolveImportPolicy(pathSpecs[0], defaults)
if len(diagnostics) != 0 || pathPolicy.required != true || pathPolicy.recursive || pathPolicy.pattern != nil {
t.Fatalf("path policy applied unsupported defaults: %#v, %#v", pathPolicy, diagnostics)
}
defaulted, diagnostics := resolveImportPolicy(omittedSpecs[0], defaults)
if len(diagnostics) != 0 || len(defaulted.pattern) != 1 || defaulted.pattern[0] != "*.yaml" {
t.Fatalf("directory pattern default = %#v, diagnostics = %#v", defaulted.pattern, diagnostics)
}
snapshot := snapshotImportDefaults(defaults)
*defaults.required = false
defaults.pattern[0] = "*.toml"
if snapshot.required == nil || !*snapshot.required || snapshot.pattern[0] != "*.yaml" {
t.Fatalf("snapshot shares mutable inputs: %#v", snapshot)
}
empty := snapshotImportDefaults(importDefaults{pattern: []string{}})
if empty.pattern == nil || len(empty.pattern) != 0 {
t.Fatalf("snapshot lost explicit empty pattern: %#v", empty)
}
if snapshotImportDefaults(importDefaults{}).pattern != nil {
t.Fatal("snapshot changed omitted pattern into explicit empty")
}
}
func TestInvalidImportDefaultPatternsFailBeforeReadingSource(t *testing.T) {
t.Parallel()
reader := &countingImportSpecReader{Reader: strings.NewReader("not: read\n")}
_, err := Load(Reader("invalid-default-pattern.yml", reader), testSecretBinding(), Options[struct{}]{
importDefaults: importDefaults{pattern: []string{"["}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != CodeOptionsInvalid {
t.Fatalf("load error = %T %v, want invalid-options LoadError", err, err)
}
if reader.reads != 0 {
t.Fatalf("reader consumed %d bytes before invalid defaults failed", reader.reads)
}
}
func parseImportSpecsYAML(t *testing.T, declarations string) ([]importDeclarationSpec, []Diagnostic) {
t.Helper()
parsed, diagnostics := parseSource(Bytes("import specification fixture", []byte("configkit:\n imports:\n"+declarations)), "source-1", defaultParseLimits())
if len(diagnostics) != 0 {
return nil, diagnostics
}
index := documentEntryIndex(parsed.document.Mapping, directiveKey)
if index < 0 {
t.Fatal("missing root directive in test fixture")
}
return parseImportSpecificationDirective(parsed.document.Mapping[index].Value, Keys(directiveKey))
}
type countingImportSpecReader struct {
io.Reader
reads int
}
func (reader *countingImportSpecReader) Read(buffer []byte) (int, error) {
read, err := reader.Reader.Read(buffer)
reader.reads += read
return read, err
}

View File

@@ -35,6 +35,7 @@ func defaultImportLimits() importLimits {
type importRoute struct { type importRoute struct {
canonical string canonical string
locations []Location locations []Location
key importNodeKey
} }
type importAccounting struct { type importAccounting struct {
@@ -45,11 +46,13 @@ type importAccounting struct {
} }
type importTraversal struct { type importTraversal struct {
access *filesystemAccess access *filesystemAccess
accounting *importAccounting accounting *importAccounting
limits importLimits limits importLimits
rule *VersionRule rule *VersionRule
versionState *sourceVersionState versionState *sourceVersionState
directiveContext rootDirectiveContext
defaults importDefaults
scope acquisitionScope scope acquisitionScope
itemScope bool itemScope bool
@@ -59,6 +62,7 @@ type importTraversal struct {
active map[resourceIdentity]int active map[resourceIdentity]int
visited map[resourceIdentity]importRoute visited map[resourceIdentity]importRoute
stack []importRoute stack []importRoute
importGraph importInclusionGraph
} }
func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -78,12 +82,16 @@ func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemC
} }
func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) { func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedDocumentsTrackedWithProcessingDeferred(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing, policy, false)
}
func loadImportedDocumentsTrackedWithProcessingDeferred(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy, deferAssembly bool) (*composedDocument, []SourceInfo, []Diagnostic) {
if nativeContext(source, context) { if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context) access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil { if diagnostic != nil {
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
} }
return loadImportedWithAccessTrackedAndProcessingPolicy(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy) return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, deferAssembly)
} }
rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
if diagnostic != nil { if diagnostic != nil {
@@ -98,7 +106,7 @@ func loadImportedDocumentsTrackedWithProcessing(source Source, context *Filesyst
if hostToken != nil { if hostToken != nil {
access.token = hostToken access.token = hostToken
} }
return loadImportedWithAccessTrackedAndProcessingPolicy(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy) return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, deferAssembly)
} }
func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) { func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -122,6 +130,10 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa
} }
func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) { func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing, policy, false)
}
func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult, policy sequencePolicy, deferAssembly bool) (*composedDocument, []SourceInfo, []Diagnostic) {
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
_ = access.close() _ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -139,22 +151,34 @@ func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess,
if mainPath != "" { if mainPath != "" {
mainInfo.Path = mainPath mainInfo.Path = mainPath
} }
mainKey := access.identity(mainPath) mainResource := access.identity(mainPath)
mainNodeKey := importKeyFor(access, mainPath, mainPath == "")
if mainPath == "" { if mainPath == "" {
mainKey = access.syntheticIdentity() mainResource = access.syntheticIdentity()
} }
if previous, repeated := visited[mainKey]; repeated { if previous, repeated := visited[mainResource]; repeated {
_ = access.close() _ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})} return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})}
} }
accounting.sources++ accounting.sources++
accounting.bytes += main.bytes accounting.bytes += main.bytes
accounting.nodes += countInputNodes(main.document) accounting.nodes += countInputNodes(main.document)
defaults := importDefaults{}
if versionState != nil {
defaults = snapshotImportDefaults(versionState.importDefaults)
}
directiveContext := rootDirectiveContextImportsOnly
if processing != nil && processing.directiveContext != 0 {
directiveContext = processing.directiveContext
}
state := importTraversal{ state := importTraversal{
access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState, access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState,
scope: acquisitionScope{destination: destination.clone()}, directiveContext: directiveContext,
sources: []SourceInfo{mainInfo}, defaults: defaults,
active: make(map[resourceIdentity]int), visited: visited, scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()},
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited,
importGraph: newImportInclusionGraph(),
} }
if mainIsItem { if mainIsItem {
state.itemScope = true state.itemScope = true
@@ -176,22 +200,28 @@ func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess,
mainIdentity = "<selected-source>" mainIdentity = "<selected-source>"
} }
mainRoute := importRoute{canonical: mainIdentity} mainRoute := importRoute{canonical: mainIdentity}
state.active[mainKey] = 0 state.importGraph.addNode(importGraphNode{key: mainNodeKey, canonical: mainIdentity, sourceID: mainInfo.ID, role: importRoleMain})
state.visited[mainKey] = mainRoute mainRoute.key = mainNodeKey
state.active[mainResource] = 0
state.visited[mainResource] = mainRoute
state.stack = append(state.stack, mainRoute) state.stack = append(state.stack, mainRoute)
var body *document.Node var body *document.Node
var declarations []importDeclaration var declarations []importDeclarationSpec
var sequenceDeclarations []sequenceDeclarationSpec
var diagnostics []Diagnostic var diagnostics []Diagnostic
if processing != nil { if processing != nil {
body = processing.payload body = processing.payload
declarations = processing.imports declarations = processing.imports
sequenceDeclarations = cloneSequenceDeclarations(processing.sequences)
} else { } else {
body, declarations, diagnostics = splitImportDirectives(main.document) body, declarations, diagnostics = splitImportDirectives(main.document)
} }
if len(diagnostics) == 0 { if len(diagnostics) == 0 {
if mainIsItem { if mainIsItem {
state.scope.contributions = append(state.scope.contributions, acquiredContribution{ state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), payload: body, document: body, processing: processing, source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain,
scopeOwner: mainNodeKey, payload: body, document: body, processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
}) })
diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{}) diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{})
} else { } else {
@@ -205,12 +235,17 @@ func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess,
accounting.nodes += placementNodes accounting.nodes += placementNodes
state.scope.contributions = append(state.scope.contributions, acquiredContribution{ state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), payload: body, document: placed, source: cloneSourceInfo(mainInfo), payload: body, document: placed,
identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey,
destination: destination.clone(), processing: processing, destination: destination.clone(), processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
}) })
diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) diagnostics = state.walkDeclarations(basePath, declarations, 0, destination)
} }
} }
} }
if len(diagnostics) == 0 {
state.scheduleAcquisitionScopes(&state.scope)
}
closeErr := access.close() closeErr := access.close()
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
@@ -222,20 +257,86 @@ func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess,
if closeErr != nil { if closeErr != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
} }
acquisition := state.scope
if mainIsItem { if mainIsItem {
localScope := state.scope localScope := state.scope
acquisition = acquisitionScope{
owner: mainNodeKey, destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), pendingItem: &localScope,
}},
}
}
if deferAssembly {
if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
}
return &composedDocument{
acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}, cloneSourceInfos(state.sources), nil
}
acquired := &composedDocument{
acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}
input := assembledInput{
parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition),
sources: state.sources, versionState: versionState, accounting: accounting,
}
composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy)
return composed, cloneSourceInfos(state.sources), diagnostics
}
// composeAcquiredInputWithPolicy is the per-input adapter from retained
// acquisition scopes to the established strict-composition behavior. It uses
// the already acquired documents and graph; source handles are closed before
// this handoff.
func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
return input.composition, nil
}
acquired := input.composition
if len(acquired.acquisition) != 1 {
return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)}
}
scope := cloneAcquisitionScope(acquired.acquisition[0])
var rule *VersionRule
if input.versionState != nil {
rule = input.versionState.effectiveRule
}
accounting := input.accounting
if accounting == nil {
accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes}
}
traversal := importTraversal{
accounting: accounting,
limits: defaultImportLimits(), rule: rule, versionState: input.versionState,
importGraph: acquired.importGraph.clone(),
}
if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil {
main := scope.contributions[0]
localScope := cloneAcquisitionScope(*main.pendingItem)
if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
}
documents := make([]*document.Node, 0, len(localScope.contributions)) documents := make([]*document.Node, 0, len(localScope.contributions))
for _, contribution := range localScope.contributions { for _, contribution := range localScope.contributions {
documents = append(documents, contribution.document) documents = append(documents, contribution.document)
} }
completed, failures := composeDocuments(documents...) completed, diagnostics := composeDocumentsWithCollectionPolicy(
if len(failures) != 0 { policy, localScope.collectionPolicies,
return nil, cloneSourceInfos(state.sources), prependCompositionDiagnosticPaths(failures, destination.Index(0)) acquiredSequenceDeclarations(localScope), documents...,
)
if len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
} }
if versionState != nil { if input.versionState != nil {
protectionRule := versionState.localRule protectionRule := input.versionState.localRule
if versionState.mode == sourceVersionModeDefault && protectionRule != nil { if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil {
prefix := destination.Index(0) prefix := main.destination.Index(0)
relative, inside := relativeHistoryPath(prefix, protectionRule.Path) relative, inside := relativeHistoryPath(prefix, protectionRule.Path)
if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) { if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) {
protectionRule = nil protectionRule = nil
@@ -245,56 +346,111 @@ func loadImportedWithAccessTrackedAndProcessingPolicy(access *filesystemAccess,
protectionRule = &scoped protectionRule = &scoped
} }
} }
if diagnostic := importedVersionClaim(documents, completed, protectionRule, 0); diagnostic != nil { if diagnostic := importedVersionClaim(documents, localScope, completed, protectionRule); diagnostic != nil {
if rule != nil { if rule != nil {
diagnostic.Path = rule.Path.clone() diagnostic.Path = rule.Path.clone()
} }
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} return nil, []Diagnostic{*diagnostic}
} }
} }
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)}
projectNestedEvidence(completed, localScope) projectNestedEvidence(completed, localScope)
_, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes) _, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil { if err != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
} }
placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem) if !main.placementCharged {
if accounting.nodes+placementNodes > limits.nodes { placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem)
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} if accounting.nodes+placementNodes > traversal.limits.nodes {
return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)}
}
accounting.nodes += placementNodes
} }
accounting.nodes += placementNodes composition, diagnostics := composeDocumentsWithPolicy(policy, placed)
composed, failures := composeDocumentsWithPolicy(policy, placed) if len(diagnostics) != 0 {
if composed != nil { return nil, diagnostics
synthetic := acquisitionScope{contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed,
destination: destination.clone(), processing: processing, nested: completed,
}}}
projectNestedEvidence(composed, synthetic)
projectAcquiredDestinations(composed, synthetic, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes
} }
return composed, cloneSourceInfos(state.sources), failures synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{
source: cloneSourceInfo(main.source), payload: completed.root, document: placed,
identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner,
destination: main.destination.clone(), processing: main.processing, nested: completed,
}}}
projectNestedEvidence(composition, synthetic)
projectAcquiredDestinations(composition, synthetic, input.sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = acquired.importGraph.clone()
return composition, nil
} }
documents := make([]*document.Node, 0, len(state.scope.contributions)) return composeAcquiredScopeWithPolicy(
for _, contribution := range state.scope.contributions { &scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting,
)
}
// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group
// handoffs. Lower policy is carried separately from registered application
// constraints; current strict composition continues to use its existing rules.
func composeAcquiredScopeWithPolicy(
scope *acquisitionScope,
lower collectionPolicyScope,
policy sequencePolicy,
graph importInclusionGraph,
sources []SourceInfo,
rule *VersionRule,
versionState *sourceVersionState,
accounting *importAccounting,
) (*composedDocument, []Diagnostic) {
traversal := importTraversal{
accounting: accounting, limits: defaultImportLimits(), rule: rule,
versionState: versionState, importGraph: graph.clone(),
}
if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 {
return nil, diagnostics
}
documents := make([]*document.Node, 0, len(scope.contributions))
for _, contribution := range scope.contributions {
documents = append(documents, contribution.document) documents = append(documents, contribution.document)
} }
composed, diagnostics := composeDocumentsWithPolicy(policy, documents...) composition, diagnostics := composeDocumentsWithCollectionPolicy(
if composed != nil { policy, lower, acquiredSequenceDeclarations(*scope), documents...,
if diagnostic := importedVersionClaim(documents, composed, rule, 0); diagnostic != nil { )
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} if len(diagnostics) != 0 {
return nil, diagnostics
}
if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil {
return nil, []Diagnostic{*diagnostic}
}
projectNestedEvidence(composition, *scope)
projectAcquiredDestinations(composition, *scope, sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = graph.clone()
return composition, nil
}
func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic {
if scope == nil {
return nil
}
for index := range scope.contributions {
contribution := &scope.contributions[index]
if contribution.pendingItem == nil {
continue
}
placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > limits.nodes {
diagnostic := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)
return &diagnostic
}
accounting.nodes += placementNodes
contribution.placementCharged = true
if diagnostic := reservePendingItemPlacements(contribution.pendingItem, limits, accounting); diagnostic != nil {
return diagnostic
} }
} }
if composed != nil { return nil
projectNestedEvidence(composed, state.scope)
projectAcquiredDestinations(composed, state.scope, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)}
composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes
}
return composed, cloneSourceInfos(state.sources), diagnostics
} }
func (s *importTraversal) checkInitialLimits() *Diagnostic { func (s *importTraversal) checkInitialLimits() *Diagnostic {
@@ -313,62 +469,77 @@ func (s *importTraversal) checkInitialLimits() *Diagnostic {
return nil return nil
} }
func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int, parentDestination Path) []Diagnostic { func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclarationSpec, depth int, parentDestination Path) []Diagnostic {
for _, declaration := range declarations { policies := make([]resolvedImportPolicy, len(declarations))
childDestination := appendDestination(parentDestination, declaration.destination) for index, spec := range declarations {
if declaration.glob != "" { policy, diagnostics := resolveImportPolicy(spec, s.defaults)
matches, diagnostics := s.expandImportGlob(basePath, declaration) if len(diagnostics) != 0 {
if len(diagnostics) != 0 { return diagnostics
}
if policy.placement == placementModeSequenceItem && len(appendDestination(parentDestination, policy.destination).segments) == 0 {
site := spec.fieldSites["placement"]
return []Diagnostic{{Code: CodeImportInvalid, Severity: SeverityError, Path: site.path.clone(), Message: "sequence-item placement requires a nonempty destination", Primary: cloneLocation(&site.location)}}
}
policies[index] = policy
}
for index, spec := range declarations {
policy := policies[index]
var selection importSelection
var diagnostics []Diagnostic
switch spec.selector {
case importSelectorPath:
selection, diagnostics = selectNamedImportCandidates(s.access, basePath, spec, policy)
case importSelectorDirectory:
selection, diagnostics = selectDirectoryImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting)
case importSelectorGlob:
selection, diagnostics = selectGlobImportCandidates(s.access, basePath, spec, policy, s.limits, s.accounting)
default:
location := spec.location
return []Diagnostic{importDiagnostic(CodeImportInvalid, "import selector is invalid", &location)}
}
if len(diagnostics) != 0 {
return diagnostics
}
childDestination := appendDestination(parentDestination, policy.destination)
// Count admitted records rather than lexical candidates: optional missing
// paths are omitted during admission, and a partially present list is not empty.
sourcesBefore := len(s.sources)
for _, candidate := range selection.candidates {
if diagnostics := s.visitPath(candidate, spec, policy, childDestination, policy.destination, depth+1); len(diagnostics) != 0 {
return diagnostics return diagnostics
} }
if len(matches) == 0 {
if declaration.optional {
continue
}
location := declaration.location
return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)}
}
for _, match := range matches {
matched := declaration
matched.path = match
matched.glob = ""
if diagnostics := s.visitPath(basePath, matched, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 {
return diagnostics
}
}
continue
} }
if diagnostics := s.visitPath(basePath, declaration, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 { if !policy.required && len(s.sources) == sourcesBefore {
return diagnostics s.recordEmptyImportSelection(spec)
} }
} }
return nil return nil
} }
func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic { func (s *importTraversal) recordEmptyImportSelection(spec importDeclarationSpec) {
location := declaration.location for index := range s.sources {
if declaration.placement == placementModeSequenceItem && len(destination.segments) == 0 { if s.sources[index].ID != spec.location.SourceID {
return []Diagnostic{{ continue
Code: CodeImportInvalid, Severity: SeverityError, }
Path: declaration.placementPath.clone(), s.sources[index].importSelections = append(s.sources[index].importSelections, importSelectionEvidence{
Message: "sequence-item placement requires a nonempty destination", path: spec.path.clone(), location: spec.location,
Primary: cloneLocation(declaration.placementLocation), selector: spec.selector, outcome: importSelectionNone,
}} })
} return
if depth > s.limits.depth {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
}
if !validImportPath(declaration.path) {
return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)}
} }
}
candidate := s.access.join(basePath, declaration.path) func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec importDeclarationSpec, policy resolvedImportPolicy, destination, relativeDestination Path, depth int) []Diagnostic {
if !s.access.contains(candidate) { location := candidate.site.location
if location.SourceID == "" && location.Line == 0 && location.Column == 0 {
location = spec.location
}
if !s.access.contains(candidate.accessPath) {
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)} return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)}
} }
canonical, err := s.access.canonical(candidate) canonical, err := s.access.canonical(candidate.accessPath)
if err != nil { if err != nil {
if declaration.optional && errors.Is(err, fs.ErrNotExist) { if !policy.required && errors.Is(err, fs.ErrNotExist) {
return nil return nil
} }
code := CodeImportInvalid code := CodeImportInvalid
@@ -383,14 +554,24 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
if !s.access.contains(canonical) { if !s.access.contains(canonical) {
return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)} return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)}
} }
if depth > s.limits.depth {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
}
currentRoute := s.currentRoute(location) currentRoute := s.currentRoute(location)
identity := s.access.identity(canonical) identity := s.access.identity(canonical)
nodeKey := importKeyFor(s.access, canonical, false)
importerKey := s.currentImportNode()
if activeIndex, ok := s.active[identity]; ok { if activeIndex, ok := s.active[identity]; ok {
closingEdge := s.importGraph.addEdge(importerKey, nodeKey, location)
if diagnostic, ok := s.importGraph.scheduleCycle(closingEdge, location); ok {
return []Diagnostic{diagnostic}
}
cycle := append([]importRoute(nil), s.stack[activeIndex:]...) cycle := append([]importRoute(nil), s.stack[activeIndex:]...)
return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)} return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)}
} }
if previous, ok := s.visited[identity]; ok { if previous, ok := s.visited[identity]; ok {
s.importGraph.addEdge(importerKey, nodeKey, location)
return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)} return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)}
} }
if s.accounting.sources+1 > s.limits.sources { if s.accounting.sources+1 > s.limits.sources {
@@ -402,13 +583,16 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
} }
sourceID := "source-" + strconv.Itoa(s.accounting.sources+1) sourceID := "source-" + strconv.Itoa(s.accounting.sources+1)
kind := SourceKindFile kind := SourceKindFile
label := declaration.path label := candidate.label
if label == "" {
label = candidate.path
}
if s.access.native != nil { if s.access.native != nil {
kind = SourceKindFilesystem kind = SourceKindFilesystem
label = s.access.label + "/" + canonical label = s.access.label + "/" + canonical
} }
info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location}
if declaration.placement == placementModeSequenceItem { if policy.placement == placementModeSequenceItem {
info.placementMode = placementModeSequenceItem info.placementMode = placementModeSequenceItem
} }
// A source's own item placement still has an exact sequence-container // A source's own item placement still has an exact sequence-container
@@ -419,7 +603,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
info.namespace = s.access.token info.namespace = s.access.token
} }
s.sources = append(s.sources, info) s.sources = append(s.sources, info)
data, skipped, diagnostic := s.readImport(canonical, sourceID, declaration.optional) data, skipped, diagnostic := s.readImport(canonical, sourceID, policy.required)
if diagnostic != nil { if diagnostic != nil {
return []Diagnostic{*diagnostic} return []Diagnostic{*diagnostic}
} }
@@ -442,37 +626,55 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
s.accounting.nodes += nodeCount s.accounting.nodes += nodeCount
var body *document.Node var body *document.Node
var declarations []importDeclaration var declarations []importDeclarationSpec
var sequenceDeclarations []sequenceDeclarationSpec
var diagnostics []Diagnostic var diagnostics []Diagnostic
var processing *sourceVersionProcessingResult var processing *sourceVersionProcessingResult
directiveContext := s.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault { if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault {
versionDestination := destination versionDestination := destination
if s.itemScope { if s.itemScope {
versionDestination = appendDestination(s.itemPrefix.Index(0), destination) versionDestination = appendDestination(s.itemPrefix.Index(0), destination)
} }
processed, failures := s.versionState.process(root, sourceVersionContribution, versionDestination, sourceID, declaration.placement) processed, failures := s.versionState.processWithDirectiveContext(root, sourceVersionContribution, versionDestination, sourceID, policy.placement, directiveContext)
if len(failures) != 0 { if len(failures) != 0 {
return failures return failures
} }
processing = &processed processing = &processed
body, declarations = processed.payload, processed.imports body, declarations = processed.payload, processed.imports
} else { } else {
body, declarations, diagnostics = splitImportDirectives(root) versionDestination := destination
if len(diagnostics) != 0 { if s.itemScope {
return diagnostics versionDestination = appendDestination(s.itemPrefix.Index(0), destination)
} }
processed := processSourceVersionWithDirectiveContext(root, s.rule, sourceVersionContribution, versionDestination, sourceVersionModeDefault, directiveContext)
if len(processed.diagnostics) != 0 {
return processed.diagnostics
}
processing = &processed
body, declarations = processed.payload, processed.imports
} }
sequenceDeclarations = cloneSequenceDeclarations(processing.sequences)
placementDestination := destination placementDestination := destination
mode := placementModeMapping mode := placementModeMapping
if declaration.placement == placementModeSequenceItem { if policy.placement == placementModeSequenceItem {
mode = placementModeSequenceItem mode = placementModeSequenceItem
} }
role := importRoleMapping
if mode == placementModeSequenceItem { if mode == placementModeSequenceItem {
route := importRoute{canonical: canonical, locations: currentRoute.locations} role = importRoleItem
}
s.importGraph.addNode(importGraphNode{key: nodeKey, canonical: canonical, sourceID: sourceID, role: role})
s.importGraph.addEdge(importerKey, nodeKey, location)
if mode == placementModeSequenceItem {
route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey}
s.active[identity] = len(s.stack) s.active[identity] = len(s.stack)
s.visited[identity] = route s.visited[identity] = route
s.stack = append(s.stack, route) s.stack = append(s.stack, route)
diagnostics := s.visitItemSource(canonical, declaration, destination, relativeDestination, depth, info, body, declarations, processing) diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, sequenceDeclarations, processing)
delete(s.active, identity) delete(s.active, identity)
s.stack = s.stack[:len(s.stack)-1] s.stack = s.stack[:len(s.stack)-1]
return diagnostics return diagnostics
@@ -486,10 +688,12 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
} }
s.accounting.nodes += placementNodes s.accounting.nodes += placementNodes
route := importRoute{canonical: canonical, locations: currentRoute.locations} route := importRoute{canonical: canonical, locations: currentRoute.locations, key: nodeKey}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{ s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: body, document: placed, source: cloneSourceInfo(info), payload: body, document: placed,
identity: nodeKey, role: role, scopeOwner: s.scope.owner,
destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
}) })
s.active[identity] = len(s.stack) s.active[identity] = len(s.stack)
s.visited[identity] = route s.visited[identity] = route
@@ -500,6 +704,17 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return diagnostics return diagnostics
} }
func (s *importTraversal) currentImportNode() importNodeKey {
if len(s.stack) != 0 {
return s.stack[len(s.stack)-1].key
}
key := s.scope.owner
if _, exists := s.importGraph.index[key]; !exists {
s.importGraph.addNode(importGraphNode{key: key, canonical: "<selected-source>", role: importRoleMain})
}
return key
}
func (s *importTraversal) sourceDestination(destination Path) Path { func (s *importTraversal) sourceDestination(destination Path) Path {
if !s.itemScope { if !s.itemScope {
return destination.clone() return destination.clone()
@@ -509,10 +724,10 @@ func (s *importTraversal) sourceDestination(destination Path) Path {
return s.itemFallback.clone() return s.itemFallback.clone()
} }
func (s *importTraversal) visitItemSource(canonical string, declaration importDeclaration, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclaration, processing *sourceVersionProcessingResult) []Diagnostic { func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, sequenceDeclarations []sequenceDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic {
parentScope := s.scope parentScope := s.scope
parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback
s.scope = acquisitionScope{} s.scope = acquisitionScope{owner: identity}
s.itemScope = true s.itemScope = true
if parentItemScope { if parentItemScope {
s.itemPrefix = appendDestination(parentPrefix, destination) s.itemPrefix = appendDestination(parentPrefix, destination)
@@ -522,47 +737,104 @@ func (s *importTraversal) visitItemSource(canonical string, declaration importDe
} }
s.scope.contributions = append(s.scope.contributions, acquiredContribution{ s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: body, document: body, source: cloneSourceInfo(info), payload: body, document: body,
processing: processing, identity: identity, role: importRoleItem, scopeOwner: identity,
processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
}) })
diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{}) diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{})
itemScope := s.scope itemScope := s.scope
s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback s.scope, s.itemScope, s.itemPrefix, s.itemFallback = parentScope, parentItemScope, parentPrefix, parentFallback
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, parentScope, destination) return diagnostics
}
documents := make([]*document.Node, 0, len(itemScope.contributions))
for _, contribution := range itemScope.contributions {
documents = append(documents, contribution.document)
}
completed, diagnostics := composeDocuments(documents...)
if len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, parentScope, destination)
} }
itemScope.destination = Path{} itemScope.destination = Path{}
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(itemScope)}
projectNestedEvidence(completed, itemScope)
pl := destination
_, placed, err := placeDocumentWithMode(completed.root, pl, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &declaration.location)}
}
placementNodes := destinationWrapperNodeCountForMode(pl, placementModeSequenceItem)
if s.accounting.nodes+placementNodes > s.limits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &declaration.location)}
}
s.accounting.nodes += placementNodes
s.scope.contributions = append(s.scope.contributions, acquiredContribution{ s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: completed.root, document: placed, source: cloneSourceInfo(info), payload: body,
destination: pl.clone(), relativeDestination: relativeDestination.clone(), processing: processing, identity: identity, role: importRoleItem, scopeOwner: parentScope.owner,
nested: completed, destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
pendingItem: &itemScope,
}) })
return nil return nil
} }
func importedVersionClaim(documents []*document.Node, composed *composedDocument, rule *VersionRule, mainIndex int) *Diagnostic { func (s *importTraversal) scheduleAcquisitionScopes(scope *acquisitionScope) {
*scope = scheduleAcquisitionScope(*scope, s.importGraph)
for index := range scope.contributions {
if pending := scope.contributions[index].pendingItem; pending != nil {
s.scheduleAcquisitionScopes(pending)
}
}
}
func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnostic {
for index := range scope.contributions {
contribution := &scope.contributions[index]
if contribution.pendingItem == nil {
continue
}
itemScope := contribution.pendingItem
if diagnostics := s.composePendingItems(itemScope); len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index)
}
documents := make([]*document.Node, 0, len(itemScope.contributions))
for _, nested := range itemScope.contributions {
documents = append(documents, nested.document)
}
// An item scope is below a sequence index. Application selectors match
// mapping-only effective paths, so none can apply inside this scope.
// Its YAML declarations remain relative to the owning record.
completed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, itemScope.collectionPolicies,
acquiredSequenceDeclarations(*itemScope), documents...,
)
if len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index)
}
// Version ownership is checked by the enclosing composition once this
// item has its effective coordinates, not against its local field names.
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(*itemScope)}
projectNestedEvidence(completed, *itemScope)
_, placed, err := placeDocumentWithMode(completed.root, contribution.destination, placementModeSequenceItem, contribution.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", contribution.source.ImportedFrom)}
}
placementNodes := destinationWrapperNodeCountForMode(contribution.destination, placementModeSequenceItem)
if !contribution.placementCharged {
if s.accounting.nodes+placementNodes > s.limits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", contribution.source.ImportedFrom)}
}
s.accounting.nodes += placementNodes
}
contribution.payload = completed.root
contribution.document = placed
contribution.nested = completed
contribution.pendingItem = nil
}
return nil
}
func acquiredSequenceDeclarations(scope acquisitionScope) []sequenceDeclarationSpec {
var declarations []sequenceDeclarationSpec
for _, contribution := range scope.contributions {
projected := cloneSequenceDeclarations(contribution.sequenceDeclarations)
for index := range projected {
projected[index].destination = appendDestination(contribution.relativeDestination, projected[index].destination)
}
declarations = append(declarations, projected...)
}
return declarations
}
func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic {
if composed == nil || rule == nil { if composed == nil || rule == nil {
return nil return nil
} }
mainIndex := -1
for index, contribution := range scope.contributions {
if contribution.role == importRoleMain {
mainIndex = index
break
}
}
ranges := indexSequenceRanges(composed.sequenceRanges) ranges := indexSequenceRanges(composed.sequenceRanges)
for index, contribution := range documents { for index, contribution := range documents {
if index == mainIndex { if index == mainIndex {
@@ -599,10 +871,12 @@ func isCompositionDiagnostic(diagnostic Diagnostic) bool {
// On a composition failure only the earlier contributions are needed to locate // On a composition failure only the earlier contributions are needed to locate
// this item. Project one enclosing scope at a time as traversal unwinds; syntax // this item. Project one enclosing scope at a time as traversal unwinds; syntax
// and access diagnostics keep their original source-local coordinates. // and access diagnostics keep their original source-local coordinates.
func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path) []Diagnostic { func projectItemDiagnosticPaths(diagnostics []Diagnostic, parent acquisitionScope, destination Path, beforeContribution int) []Diagnostic {
for _, diagnostic := range diagnostics { for _, diagnostic := range diagnostics {
if isCompositionDiagnostic(diagnostic) { if isCompositionDiagnostic(diagnostic) {
index := priorSequenceLength([]acquisitionScope{parent}, destination) prefix := parent
prefix.contributions = parent.contributions[:beforeContribution]
index := priorSequenceLength([]acquisitionScope{prefix}, destination)
return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index)) return prependCompositionDiagnosticPaths(diagnostics, destination.Index(index))
} }
} }
@@ -654,8 +928,13 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
projectedApplications = append(projectedApplications, applications[sourceIndex]) projectedApplications = append(projectedApplications, applications[sourceIndex])
continue continue
} }
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start itemIndex := 0
prefix := contribution.destination.Index(start) if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
prefix := contribution.destination.Index(itemIndex)
if sourceIndex < len(applications) { if sourceIndex < len(applications) {
outer := applications[sourceIndex] outer := applications[sourceIndex]
structural := document.Clone(outer.value) structural := document.Clone(outer.value)
@@ -677,6 +956,71 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix) projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix)
} }
composed.applications = projectedApplications composed.applications = projectedApplications
projectNestedCollectionPolicies(composed, scope, ranges)
}
func projectNestedCollectionPolicies(composed *composedDocument, scope acquisitionScope, ranges map[sequenceRangeKey]sequenceRange) {
if composed == nil {
return
}
type itemPolicyProjection struct {
destination Path
policy collectionPolicy
base int
}
projected := make(map[string]*itemPolicyProjection)
var prefixLengths map[string]int
for sourceIndex, contribution := range scope.contributions {
if prefixLengths != nil && sourceIndex > 0 {
addSequencePrefixLengths(scope.contributions[sourceIndex-1].document, Path{}, prefixLengths)
}
if contribution.nested == nil || len(contribution.nested.collectionPolicies.collections) == 0 {
continue
}
key := contribution.destination.String()
assigned := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: key}]
itemIndex, found := assigned.outputForInput(0)
if !found {
// Normal composition supplies the assignment directly. For a
// handoff without ranges, scan each preceding tree only once.
if prefixLengths == nil {
prefixLengths = make(map[string]int)
for _, previous := range scope.contributions[:sourceIndex] {
addSequencePrefixLengths(previous.document, Path{}, prefixLengths)
}
}
itemIndex = prefixLengths[key]
}
working, initialized := projected[key]
if !initialized {
// Read one owned copy per destination, then add each incoming
// record without recopying earlier records and their scopes.
policy, _ := composed.collectionPolicies.at(contribution.destination)
working = &itemPolicyProjection{destination: contribution.destination, policy: policy, base: len(policy.recordOrder)}
projected[key] = working
}
working.policy.setRecordPolicyAt(working.base+itemIndex, collectionRecordPolicy{
owner: newCollectionRecordPolicy().owner,
scope: contribution.nested.collectionPolicies,
})
}
for _, working := range projected {
composed.collectionPolicies.set(working.destination, working.policy)
}
}
func addSequencePrefixLengths(node *document.Node, path Path, lengths map[string]int) {
if node == nil {
return
}
if node.Kind == document.Sequence {
lengths[path.String()] += len(node.Sequence)
return
}
// Contribution destinations select mapping paths, not sequence members.
for _, entry := range node.Mapping {
addSequencePrefixLengths(entry.Value, path.Key(entry.Key), lengths)
}
} }
func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) { func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) {
@@ -712,7 +1056,12 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
} }
contributionPrefix := prefix contributionPrefix := prefix
contributionInsideItem := insideItem contributionInsideItem := insideItem
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start itemIndex := 0
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
if contribution.source.placementMode == placementModeSequenceItem { if contribution.source.placementMode == placementModeSequenceItem {
container := appendDestination(contributionPrefix, contribution.destination) container := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = container.clone() contribution.source.Destination = container.clone()
@@ -722,7 +1071,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
sources[position].Destination = container sources[position].Destination = container
sources[position].destinationUnresolved = false sources[position].destinationUnresolved = false
} }
contributionPrefix = appendDestination(container, Path{}.Index(start)) contributionPrefix = appendDestination(container, Path{}.Index(itemIndex))
contributionInsideItem = true contributionInsideItem = true
projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex) projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex)
} else if contributionInsideItem { } else if contributionInsideItem {
@@ -765,7 +1114,9 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des
} }
item, ok := schemaIndex(path.segments[position]) item, ok := schemaIndex(path.segments[position])
if ok && path.segments[position].kind != pathSegmentKey { if ok && path.segments[position].kind != pathSegmentKey {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: assigned.start + item} if outputIndex, projected := assigned.outputForInput(item); projected {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex}
}
} }
} }
paths[index] = path paths[index] = path
@@ -774,7 +1125,7 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des
} }
} }
func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { func (s *importTraversal) readImport(name, sourceID string, required bool) ([]byte, bool, *Diagnostic) {
remaining := s.limits.bytes - s.accounting.bytes remaining := s.limits.bytes - s.accounting.bytes
limit := min(int64(maxSourceBytes), remaining) limit := min(int64(maxSourceBytes), remaining)
if limit < 0 { if limit < 0 {
@@ -782,7 +1133,7 @@ func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]by
} }
data, failure := readRegularFile(name, s.access.stat, s.access.open, limit) data, failure := readRegularFile(name, s.access.stat, s.access.open, limit)
if failure != nil { if failure != nil {
if optional && failure.missingTarget { if !required && failure.missingTarget {
return nil, true, nil return nil, true, nil
} }
value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit") value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit")

View File

@@ -91,6 +91,80 @@ settings:
} }
} }
func TestLoadOrdinaryImportsUseDependencyReadyCompositionOrder(t *testing.T) {
type task struct {
ID string `yaml:"id"`
}
type config struct {
Tasks []task `yaml:"tasks"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: z-parent.yml
- path: a-independent.yml
tasks: [{id: main}]
`)
zParentPath := writeImportFile(t, directory, "z-parent.yml", `configkit:
imports:
- path: b-dependency.yml
tasks: [{id: z-parent}]
`)
aIndependentPath := writeImportFile(t, directory, "a-independent.yml", "tasks: [{id: a-independent}]\n")
bDependencyPath := writeImportFile(t, directory, "b-dependency.yml", "tasks: [{id: b-dependency}]\n")
input, diagnostics := assembleBaseInput[config](File(mainPath), Options[config]{
Imports: true,
sequencePolicy: mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("tasks")}, identityFields: []string{"id"}}}),
})
if len(diagnostics) != 0 {
t.Fatalf("assemble imports: %#v", diagnostics)
}
tasks, ok := documentAtPath(input.document, Keys("tasks"))
if !ok || tasks.Kind != document.Sequence {
t.Fatalf("tasks document = %#v, %v", tasks, ok)
}
wantOrder := []string{"a-independent", "b-dependency", "z-parent", "main"}
if len(tasks.Sequence) != len(wantOrder) {
t.Fatalf("tasks = %#v", tasks.Sequence)
}
for index, want := range wantOrder {
id, found := documentAtPath(tasks, Path{}.Index(index).Key("id"))
if !found || id.Text != want {
t.Errorf("task %d ID = %#v, want %q", index, id, want)
}
}
wantAcquisition := []string{
canonicalTestPath(t, mainPath), canonicalTestPath(t, zParentPath),
canonicalTestPath(t, bDependencyPath), canonicalTestPath(t, aIndependentPath),
}
for index, want := range wantAcquisition {
if got := input.sources[index].Path; got != want {
t.Errorf("acquired source %d = %q, want %q", index, got, want)
}
}
}
func TestAcquiredContextsUseSelectedFileWhenSourceRecordsAreReordered(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "settings: {root: main}\n")
importedPath := writeImportFile(t, directory, "nested/imported.yml", "settings: {nested: imported}\n")
sources := []SourceInfo{
{ID: "source-2", Path: canonicalTestPath(t, importedPath), Kind: SourceKindFile},
{ID: "source-1", Path: canonicalTestPath(t, mainPath), Kind: SourceKindFile},
}
contexts, diagnostic := contextsForAcquiredSources(File(mainPath), nil, nil, sources)
if diagnostic != nil {
t.Fatalf("source contexts: %#v", diagnostic)
}
if got := contexts["source-1"].BaseDir; got != "." {
t.Errorf("main source base = %q, want root-relative current directory", got)
}
if got := contexts["source-2"].BaseDir; got != "nested" {
t.Errorf("imported source base = %q, want nested", got)
}
}
func TestLoadImportsFromProfilesDirectoryAsBaseInput(t *testing.T) { func TestLoadImportsFromProfilesDirectoryAsBaseInput(t *testing.T) {
directory := t.TempDir() directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: profiles/shared.yml}]}\n") mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: profiles/shared.yml}]}\n")
@@ -153,10 +227,11 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) {
{name: "unknown directive field", body: "configkit: {unknown: true}\n", code: CodeImportInvalid}, {name: "unknown directive field", body: "configkit: {unknown: true}\n", code: CodeImportInvalid},
{name: "imports kind", body: "configkit: {imports: {}}\n", code: CodeImportInvalid}, {name: "imports kind", body: "configkit: {imports: {}}\n", code: CodeImportInvalid},
{name: "declaration kind", body: "configkit: {imports: [file.yml]}\n", code: CodeImportInvalid}, {name: "declaration kind", body: "configkit: {imports: [file.yml]}\n", code: CodeImportInvalid},
{name: "missing target", body: "configkit: {imports: [{optional: true}]}\n", code: CodeImportInvalid}, {name: "missing target", body: "configkit: {imports: [{required: true}]}\n", code: CodeImportInvalid},
{name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, {name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid},
{name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, {name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid},
{name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, {name: "removed optional field", body: "configkit: {imports: [{path: a.yml, optional: true}]}\n", code: CodeImportInvalid},
{name: "required kind", body: "configkit: {imports: [{path: a.yml, required: 'yes'}]}\n", code: CodeImportInvalid},
{name: "placement kind", body: "configkit: {imports: [{path: a.yml, placement: true}]}\n", code: CodeImportInvalid}, {name: "placement kind", body: "configkit: {imports: [{path: a.yml, placement: true}]}\n", code: CodeImportInvalid},
{name: "placement value", body: "configkit: {imports: [{path: a.yml, placement: item}]}\n", code: CodeImportInvalid}, {name: "placement value", body: "configkit: {imports: [{path: a.yml, placement: item}]}\n", code: CodeImportInvalid},
{name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid}, {name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid},
@@ -399,9 +474,8 @@ func TestLoadItemGlobKeepsDefaultModeIndependentAndOptionalMissesAbsent(t *testi
- glob: records/*.yml - glob: records/*.yml
destination: [items] destination: [items]
placement: sequence-item placement: sequence-item
- path: absent/*.yml - glob: absent/*.yml
destination: [items] destination: [items]
optional: true
placement: sequence-item placement: sequence-item
- path: other.yml - path: other.yml
destination: [other] destination: [other]
@@ -451,6 +525,141 @@ func TestLoadMainItemPlacementKeepsItsHelpersInTheItem(t *testing.T) {
} }
} }
func TestProjectAcquiredDestinationsUsesMappedSequenceItemIndex(t *testing.T) {
child := &composedDocument{}
childScope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "nested-source"}, destination: Keys("settings"),
}}}
child.acquisition = []acquisitionScope{childScope}
composed := &composedDocument{sequenceRanges: []sequenceRange{{
sourceIndex: 0, path: Keys("records"), start: 0, end: 1,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}}}
scope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "outer-source", placementMode: placementModeSequenceItem},
destination: Keys("records"), nested: child,
}}}
sources := []SourceInfo{{ID: "outer-source"}, {ID: "nested-source"}}
projectAcquiredDestinations(composed, scope, sources)
if got, want := sources[0].Destination.String(), "/records"; got != want {
t.Errorf("outer source destination = %q, want %q", got, want)
}
if got, want := sources[1].Destination.String(), "/records/2/settings"; got != want {
t.Errorf("nested source destination = %q, want %q", got, want)
}
}
func TestProjectNestedEvidenceUsesMappedSequenceItemIndex(t *testing.T) {
outer := historyApplication{
value: parseCompositionDocument(t, "outer-source", "records: [{id: outer}]\n"),
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}},
}
outer.sequencesByPath = indexSequenceHistoryContributions(outer.sequences)
nestedRoot := parseCompositionDocument(t, "nested-source", "settings: {value: nested}\n")
nestedOrigin := Origin{Layer: LayerBase, Locations: []Location{{SourceID: "nested-source"}}}
nested := &composedDocument{
root: nestedRoot, applications: []historyApplication{{value: nestedRoot}},
origins: map[string]Origin{"/settings/value": nestedOrigin},
}
scope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "outer-source"}, destination: Keys("records"), nested: nested,
}}}
composed := &composedDocument{
applications: []historyApplication{outer},
sequenceRanges: []sequenceRange{{
sourceIndex: 0, path: Keys("records"), start: 0, end: 1,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}},
origins: make(map[string]Origin),
}
projectNestedEvidence(composed, scope)
if got, want := composed.applications[1].path.String(), "/records/2"; got != want {
t.Fatalf("nested history application path = %q, want %q", got, want)
}
if origin := composed.origins["/records/2/settings/value"]; len(origin.Locations) != 1 || origin.Locations[0].SourceID != "nested-source" {
t.Fatalf("nested origin = %#v", origin)
}
query := Keys("records").Index(2).Key("settings").Key("value")
if operation, relevant := applicationOperationPath(composed.applications[1], query, nil); !relevant || operation.String() != query.String() {
t.Fatalf("nested historical query = %s, %v", operation.String(), relevant)
}
replayed := parseCompositionDocument(t, "lower", "records: [{id: first}, {id: middle}, {id: last}]\n")
for _, application := range composed.applications {
applyHistoryApplication(&replayed, application, new(mappingIndex))
}
if value, present := documentAtPath(replayed, query); !present || value.Text != "nested" {
t.Fatalf("replayed nested value = %#v, %v", value, present)
}
composed.applications[1].path.segments[0].key = "detached"
if got := nested.applications[0].path.String(); got != "" {
t.Fatalf("projected nested history path aliases its source: %q", got)
}
}
func TestDeferredInputCompositionUsesRetainedAcquisitionScopes(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [records]
placement: sequence-item
name: main
`)
itemPath := writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: helper.yml
id: item
`)
helperPath := writeImportFile(t, directory, "helper.yml", "value: acquired\n")
accounting := &importAccounting{}
input, diagnostics := assembleBaseInputTracked(File(mainPath), Options[struct{}]{
Imports: true, Filesystem: &FilesystemContext{Root: directory}, deferAssembly: true,
}, Path{}, accounting, nil, nil)
if len(diagnostics) != 0 {
t.Fatalf("acquire input: %#v", diagnostics)
}
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) != 1 {
t.Fatalf("acquisition handoff was composed early: %#v", input.composition)
}
if len(input.sources) != 3 || input.sources[0].ID != "source-1" || input.sources[1].ID != "source-2" || input.sources[2].ID != "source-3" {
t.Fatalf("acquired source order/identity = %#v", input.sources)
}
if _, err := os.Stat(itemPath); err != nil {
t.Fatalf("item source disappeared before composition: %v", err)
}
if _, err := os.Stat(helperPath); err != nil {
t.Fatalf("helper source disappeared before composition: %v", err)
}
composition, diagnostics := compositionForInputWithPolicy(input, sequencePolicy{})
if len(diagnostics) != 0 {
t.Fatalf("compose acquired input: %#v", diagnostics)
}
if value, present := documentAtPath(composition.root, Keys("name")); !present || value.Text != "main" {
t.Fatalf("main value = %#v, %v", value, present)
}
for path, want := range map[string]string{
"/records/0/id": "item",
"/records/0/value": "acquired",
} {
value, present := documentAtPath(composition.root, mustParsePath(t, path))
if !present || value.Text != want {
t.Errorf("composed %s = %#v, %v; want %q", path, value, present, want)
}
}
if got, want := input.sources[1].Destination.String(), "/records"; got != want {
t.Errorf("item destination = %q, want %q", got, want)
}
if got, want := input.sources[2].Destination.String(), "/records/0"; got != want {
t.Errorf("helper destination = %q, want %q", got, want)
}
}
func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) { func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) {
t.Parallel() t.Parallel()
type config struct { type config struct {
@@ -470,7 +679,7 @@ func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T)
t.Fatalf("assemble item import: %#v", diagnostics) t.Fatalf("assemble item import: %#v", diagnostics)
} }
authored := parseDocumentForDecode(t, "items: [{id: authored-id, name: authored}]\n") authored := parseDocumentForDecode(t, "items: [{id: authored-id, name: authored}]\n")
composed, diagnostics := composeDocumentsWithPolicy(sequencePolicy{identityFields: map[string]string{"/items": "id"}}, authored, input.document) composed, diagnostics := composeDocumentsWithPolicy(mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}), authored, input.document)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
t.Fatalf("compose authored and imported lists: %#v", diagnostics) t.Fatalf("compose authored and imported lists: %#v", diagnostics)
} }
@@ -504,7 +713,7 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) {
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
body := "configkit:\n imports:\n - path: child.yml\n" + test.destination body := "configkit:\n imports:\n - path: child.yml\n" + test.destination
declarations := parseImportDeclarationsYAML(t, body) declarations := parseImportSpecificationsYAML(t, body)
if len(declarations) != 1 { if len(declarations) != 1 {
t.Fatalf("declarations = %#v", declarations) t.Fatalf("declarations = %#v", declarations)
} }
@@ -523,7 +732,7 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) {
func TestParseImportPlacementDefaultsPerDeclaration(t *testing.T) { func TestParseImportPlacementDefaultsPerDeclaration(t *testing.T) {
t.Parallel() t.Parallel()
declarations := parseImportDeclarationsYAML(t, `configkit: declarations := parseImportSpecificationsYAML(t, `configkit:
imports: imports:
- path: mapping.yml - path: mapping.yml
- glob: records/*.yml - glob: records/*.yml
@@ -555,7 +764,7 @@ func TestParseImportDestinationRejectsInvalidShapesAndMembers(t *testing.T) {
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
body := "configkit:\n imports:\n - path: child.yml\n destination: " + test.value + "\n" body := "configkit:\n imports:\n - path: child.yml\n destination: " + test.value + "\n"
parsed, diagnostics := parseImportDeclarationYAML(body) parsed, diagnostics := parseImportSpecificationYAML(body)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid || diagnostics[0].Path.String() != test.wantPath { if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid || diagnostics[0].Path.String() != test.wantPath {
t.Fatalf("parse = %#v, want import.invalid at %q", diagnostics, test.wantPath) t.Fatalf("parse = %#v, want import.invalid at %q", diagnostics, test.wantPath)
} }
@@ -578,12 +787,12 @@ func TestImportDestinationValidationPrecedesOptionalSuppression(t *testing.T) {
}{ }{
{ {
name: "optional missing path", name: "optional missing path",
body: "configkit:\n imports:\n - path: absent.yml\n optional: true\n destination: null\n", body: "configkit:\n imports:\n - path: absent.yml\n destination: null\n",
path: "/configkit/imports/0/destination", path: "/configkit/imports/0/destination",
}, },
{ {
name: "optional unmatched glob", name: "optional unmatched glob",
body: "configkit:\n imports:\n - glob: absent/*.yml\n optional: true\n destination: [extends]\n", body: "configkit:\n imports:\n - glob: absent/*.yml\n destination: [extends]\n",
path: "/configkit/imports/0/destination/0", path: "/configkit/imports/0/destination/0",
}, },
} { } {
@@ -604,7 +813,6 @@ func TestLoadImportMissingContainmentAndRegularFileRules(t *testing.T) {
mainPath := writeImportFile(t, directory, "main.yml", `configkit: mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports: imports:
- path: missing.yml - path: missing.yml
optional: true
settings: {root: retained} settings: {root: retained}
`) `)
result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true}) result, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
@@ -637,12 +845,42 @@ settings: {root: retained}
if test.setup != nil { if test.setup != nil {
test.setup(t, directory) test.setup(t, directory)
} }
mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: \""+test.path+"\"\n") required := ""
if test.code == CodeImportMissing {
required = "\n required: true"
}
mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: \""+test.path+"\""+required+"\n")
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, test.code)
}) })
} }
} }
func TestImportDefaultsReachOrdinaryAndGroupedDeclarations(t *testing.T) {
t.Parallel()
required := true
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: child.yml}]}\n")
writeImportFile(t, directory, "child.yml", "configkit: {imports: [{path: absent.yml}]}\n")
_, err := testLoad[importTestConfig](File(mainPath), Options[importTestConfig]{
Imports: true, importDefaults: importDefaults{required: &required},
})
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != CodeImportMissing {
t.Fatalf("ordinary transitive required default error = %#v", err)
}
basePath := writeImportFile(t, directory, "base.yml", "settings: {root: base}\n")
underlayPath := writeImportFile(t, directory, "underlay.yml", "configkit: {imports: [{path: absent.yml}]}\n")
_, err = loadGroupedWithSecretLookup(loadRequest[importTestConfig]{
main: loadInput{source: File(basePath)},
underlays: []loadGroup{{inputs: []loadInput{{source: File(underlayPath), imports: true}}}},
importDefaults: importDefaults{required: &required},
}, testSecretBinding(), func(string) (string, bool) { return "", false })
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != CodeImportMissing {
t.Fatalf("grouped underlay required default error = %#v", err)
}
}
func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) { func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) {
t.Run("cycle", func(t *testing.T) { t.Run("cycle", func(t *testing.T) {
directory := t.TempDir() directory := t.TempDir()
@@ -656,6 +894,22 @@ func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) {
} }
}) })
t.Run("cycle through an item scope", func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: main.yml}]}\n")
loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportCycle)
diagnostic := loadError.Diagnostics()[0]
if diagnostic.Primary == nil || len(diagnostic.Related) != 1 || !strings.Contains(diagnostic.Message, "main.yml") || !strings.Contains(diagnostic.Message, "item.yml") {
t.Fatalf("nested item cycle diagnostic = %#v", diagnostic)
}
})
t.Run("diamond", func(t *testing.T) { t.Run("diamond", func(t *testing.T) {
directory := t.TempDir() directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports: [{path: left.yml}, {path: right.yml}]\n") mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports: [{path: left.yml}, {path: right.yml}]\n")
@@ -686,6 +940,192 @@ func TestLoadImportIdentityReportsCyclesAndRepeatedRoutes(t *testing.T) {
}) })
} }
func TestItemAcquisitionRetainsInclusionGraphAndScopeOwnership(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
itemPath := writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: helper.yml}]}\nname: item\n")
helperPath := writeImportFile(t, directory, "helper.yml", "id: supplied\n")
input, diagnostics := assembleBaseInput[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
if len(diagnostics) != 0 {
t.Fatalf("assemble input: %#v", diagnostics)
}
graph := input.composition.importGraph
if len(graph.nodes) != 3 || len(graph.edges) != 2 {
t.Fatalf("inclusion graph has %d nodes and %d edges, want 3 and 2", len(graph.nodes), len(graph.edges))
}
for index, want := range []struct {
path string
role importContributionRole
}{
{path: canonicalTestPath(t, mainPath), role: importRoleMain},
{path: canonicalTestPath(t, itemPath), role: importRoleItem},
{path: canonicalTestPath(t, helperPath), role: importRoleMapping},
} {
node := graph.nodes[index]
if node.canonical != want.path || node.role != want.role || node.sourceID != "source-"+strconv.Itoa(index+1) {
t.Errorf("graph node %d = %#v, want %s, %q, source-%d", index, node, want.role, want.path, index+1)
}
}
rootScope := input.composition.acquisition[0]
if rootScope.owner != graph.nodes[0].key || len(rootScope.contributions) != 2 {
t.Fatalf("root acquisition scope = %#v", rootScope)
}
var item *acquiredContribution
for index := range rootScope.contributions {
if rootScope.contributions[index].role == importRoleItem {
item = &rootScope.contributions[index]
break
}
}
if item == nil || item.identity != graph.nodes[1].key || item.scopeOwner != rootScope.owner || item.nested == nil {
t.Fatalf("outer item contribution = %#v", item)
}
itemScope := item.nested.acquisition[0]
if itemScope.owner != item.identity || len(itemScope.contributions) != 2 {
t.Fatalf("nested item scope = %#v", itemScope)
}
for _, contribution := range itemScope.contributions {
if contribution.scopeOwner != item.identity {
t.Errorf("nested contribution scope owner = %#v, want item %v", contribution.scopeOwner, item.identity)
}
}
if graph.edges[0].importer != graph.nodes[0].key || graph.edges[0].imported != graph.nodes[1].key || graph.edges[0].location.SourceID != "source-1" ||
graph.edges[1].importer != graph.nodes[1].key || graph.edges[1].imported != graph.nodes[2].key || graph.edges[1].location.SourceID != "source-2" {
t.Fatalf("inclusion edges = %#v", graph.edges)
}
}
func TestItemContributionsUseDependencyReadySchedules(t *testing.T) {
t.Run("outer independent items", func(t *testing.T) {
type item struct {
ID string `yaml:"id"`
}
type config struct {
Items []item `yaml:"items"`
}
definition := Definition[config]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: z-item.yml
destination: [items]
placement: sequence-item
- path: a-item.yml
destination: [items]
placement: sequence-item
items: []
`)
zPath := writeImportFile(t, directory, "z-item.yml", "id: z\n")
aPath := writeImportFile(t, directory, "a-item.yml", "id: a\n")
result, err := testLoad[config](File(mainPath), Options[config]{Imports: true, Definition: definition})
if err != nil {
t.Fatalf("load item imports: %v", err)
}
if len(result.Config.Items) != 2 || result.Config.Items[0].ID != "a" || result.Config.Items[1].ID != "z" {
t.Fatalf("scheduled items = %#v", result.Config.Items)
}
wantSources := []string{canonicalTestPath(t, mainPath), canonicalTestPath(t, zPath), canonicalTestPath(t, aPath)}
for index, want := range wantSources {
if got := result.Sources()[index].Path; got != want {
t.Errorf("acquired source %d = %q, want %q", index, got, want)
}
}
})
t.Run("unequal helper branches", func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: b-parent.yml
- path: c-independent.yml
id: item
`)
writeImportFile(t, directory, "b-parent.yml", "configkit: {imports: [{path: a-dependency.yml}]}\nid: b\n")
writeImportFile(t, directory, "a-dependency.yml", "id: a\n")
writeImportFile(t, directory, "c-independent.yml", "id: c\n")
loadError := requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeComposeConflict)
diagnostic := loadError.Diagnostics()[0]
if diagnostic.Path.String() != "/items/0/id" || diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-4" {
t.Fatalf("item helper conflict = %#v", diagnostic)
}
wantRelated := map[string]bool{"source-3": false, "source-5": false, "source-2": false}
if len(diagnostic.Related) != len(wantRelated) {
t.Fatalf("related sites = %#v", diagnostic.Related)
}
for _, site := range diagnostic.Related {
if _, expected := wantRelated[site.SourceID]; !expected {
t.Errorf("unexpected related site source %q", site.SourceID)
}
wantRelated[site.SourceID] = true
}
for sourceID, found := range wantRelated {
if !found {
t.Errorf("related sites do not include %q", sourceID)
}
}
})
t.Run("selected main item dependencies", func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: helper.yml}]}\nid: main\n")
writeImportFile(t, directory, "helper.yml", "id: helper\n")
options := Options[importTestConfig]{
Imports: true, destination: Keys("items"), placementMode: placementModeSequenceItem,
}
loadError := requireLoadCode(t, File(mainPath), options, CodeComposeConflict)
diagnostic := loadError.Diagnostics()[0]
if diagnostic.Path.String() != "/items/0/id" || diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" || len(diagnostic.Related) != 1 || diagnostic.Related[0].SourceID != "source-1" {
t.Fatalf("main item helper conflict = %#v", diagnostic)
}
})
}
func TestItemHelperScopeScheduleInterleavesUnequalBranches(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: b-parent.yml}, {path: c-independent.yml}]}\nitem: true\n")
writeImportFile(t, directory, "b-parent.yml", "configkit: {imports: [{path: a-dependency.yml}]}\nb: true\n")
writeImportFile(t, directory, "a-dependency.yml", "a: true\n")
writeImportFile(t, directory, "c-independent.yml", "c: true\n")
input, diagnostics := assembleBaseInput[importTestConfig](File(mainPath), Options[importTestConfig]{Imports: true})
if len(diagnostics) != 0 {
t.Fatalf("assemble item scope: %#v", diagnostics)
}
var nested *composedDocument
for _, contribution := range input.composition.acquisition[0].contributions {
if contribution.role == importRoleItem {
nested = contribution.nested
break
}
}
if nested == nil || len(nested.acquisition) != 1 {
t.Fatalf("item composition = %#v", nested)
}
wantIDs := []string{"source-4", "source-3", "source-5", "source-2"}
for index, want := range wantIDs {
if got := nested.acquisition[0].contributions[index].source.ID; got != want {
t.Errorf("local contribution %d = %q, want %q", index, got, want)
}
}
}
func TestLoadImportsCanonicalSymlinksAndKeepsHardLinksDistinct(t *testing.T) { func TestLoadImportsCanonicalSymlinksAndKeepsHardLinksDistinct(t *testing.T) {
if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
t.Skip("initial filesystem contract covers Linux and macOS") t.Skip("initial filesystem contract covers Linux and macOS")
@@ -797,7 +1237,7 @@ func TestImportedParseAndDecodeDiagnosticsKeepTheirSource(t *testing.T) {
func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) { func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) {
directory := t.TempDir() directory := t.TempDir()
mainPath := writeImportFile(t, directory, "outside.yml", "configkit: {imports: [{path: ../missing.yml, optional: true}]}\n") mainPath := writeImportFile(t, directory, "outside.yml", "configkit: {imports: [{path: ../missing.yml}]}\n")
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeImportOutsideRoot)
locked := writeImportFile(t, directory, "locked.yml", "settings: {root: locked}\n") locked := writeImportFile(t, directory, "locked.yml", "settings: {root: locked}\n")
@@ -809,7 +1249,7 @@ func TestOptionalImportDoesNotSuppressUnsafeOrUnreadableTargets(t *testing.T) {
_ = file.Close() _ = file.Close()
t.Skip("filesystem does not enforce the unreadable-file fixture") t.Skip("filesystem does not enforce the unreadable-file fixture")
} }
mainPath = writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: locked.yml, optional: true}]}\n") mainPath = writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: locked.yml}]}\n")
requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourcePermission) requireLoadCode(t, File(mainPath), Options[importTestConfig]{Imports: true}, CodeSourcePermission)
} }
@@ -844,6 +1284,48 @@ func TestImportTraversalEnforcesPrivateBudgets(t *testing.T) {
} }
} }
func TestImportDepthBudgetCountsSelectedEdges(t *testing.T) {
for _, test := range []struct {
name, declaration string
code Code
}{
{name: "no declaration"},
{name: "empty paths", declaration: "path: []"},
{name: "required empty paths", declaration: "path: []\n required: true"},
{name: "missing path", declaration: "path: absent.yml"},
{name: "missing directory", declaration: "dir: absent"},
{name: "missing glob", declaration: "glob: 'absent/*.yml'"},
{name: "selected path", declaration: "path: parts/next.yml", code: CodeImportLimitExceeded},
{name: "selected directory", declaration: "dir: parts", code: CodeImportLimitExceeded},
{name: "selected glob", declaration: "glob: 'parts/*.yml'", code: CodeImportLimitExceeded},
{name: "invalid empty declaration", declaration: "dir: []\n depth: 1", code: CodeImportInvalid},
{name: "invalid absent glob", declaration: "glob: '['", code: CodeImportInvalid},
} {
t.Run(test.name, func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: leaf.yml}]}\n")
leaf := "end: true\n"
if test.declaration != "" {
leaf += "configkit:\n imports:\n - " + test.declaration + "\n"
}
writeImportFile(t, directory, "leaf.yml", leaf)
writeImportFile(t, directory, "parts/next.yml", "next: true\n")
limits := defaultImportLimits()
limits.depth = 1
composed, sources, diagnostics := loadImportedDocuments(File(mainPath), nil, parseImportMain(t, mainPath), nil, limits)
if test.code != "" {
if len(diagnostics) != 1 || diagnostics[0].Code != test.code {
t.Fatalf("diagnostics = %#v, want %s", diagnostics, test.code)
}
return
}
if len(diagnostics) != 0 || composed == nil || len(sources) != 2 {
t.Fatalf("depth-boundary composition: sources=%#v diagnostics=%#v", sources, diagnostics)
}
})
}
}
func TestImportTraversalAcceptsExactPrivateBudgets(t *testing.T) { func TestImportTraversalAcceptsExactPrivateBudgets(t *testing.T) {
directory := t.TempDir() directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: imported.yml}]}\n") mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: imported.yml}]}\n")
@@ -919,16 +1401,16 @@ func parseImportMain(t *testing.T, filename string) parsedSource {
return parsed return parsed
} }
func parseImportDeclarationsYAML(t *testing.T, input string) []importDeclaration { func parseImportSpecificationsYAML(t *testing.T, input string) []importDeclarationSpec {
t.Helper() t.Helper()
declarations, diagnostics := parseImportDeclarationYAML(input) declarations, diagnostics := parseImportSpecificationYAML(input)
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
t.Fatalf("import parse diagnostics = %#v", diagnostics) t.Fatalf("import parse diagnostics = %#v", diagnostics)
} }
return declarations return declarations
} }
func parseImportDeclarationYAML(input string) ([]importDeclaration, []Diagnostic) { func parseImportSpecificationYAML(input string) ([]importDeclarationSpec, []Diagnostic) {
parsed, diagnostics := parseSource(Bytes("import fixture", []byte(input)), "source-1", defaultParseLimits()) parsed, diagnostics := parseSource(Bytes("import fixture", []byte(input)), "source-1", defaultParseLimits())
if len(diagnostics) != 0 { if len(diagnostics) != 0 {
return nil, diagnostics return nil, diagnostics

View File

@@ -1,20 +1,19 @@
package engine package engine
import ( import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
"slices"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
const inheritanceKey = "extends" const inheritanceKey = "extends"
// These private records keep structural evidence separate from acquired source // These private records keep structural evidence separate from acquired source
// records. Later inspection can project a transfer onto individual descendants. // records. Later inspection can project a transfer onto individual descendants.
type inheritanceTransfer struct { type inheritanceTransfer struct {
from Path from Path
to Path to Path
reference Path reference Path
locations []Location locations []Location
policyTemplates []collectionPolicyScope
templatePolicies collectionPolicyScope
projectedPolicies []projectedCollectionPolicy
} }
type inheritanceControl struct { type inheritanceControl struct {
@@ -24,22 +23,110 @@ type inheritanceControl struct {
} }
type inheritanceScope struct { type inheritanceScope struct {
root *document.Node root *document.Node
origins map[string]Origin origins map[string]Origin
transfers []inheritanceTransfer collectionPolicies collectionPolicyScope
controls []inheritanceControl transfers []inheritanceTransfer
replacements []inheritanceReplacement controls []inheritanceControl
operations []inheritanceOperation replacements []inheritanceReplacement
operations []inheritanceOperation
} }
type inheritanceOperation struct { type inheritanceOperation struct {
control *inheritanceControl control *inheritanceControl
transfer *inheritanceTransfer transfer *inheritanceTransfer
controlOrigin Origin controlOrigin Origin
parent *document.Node parent *document.Node
effective *document.Node effective *document.Node
parentOrigins map[string]Origin parentOrigins map[string]Origin
effectiveOrigins map[string]Origin effectiveOrigins map[string]Origin
parentPolicies []collectionPolicyScope
templatePolicies collectionPolicyScope
effectivePolicies []collectionPolicyScope
sequences []sequenceHistoryContribution
identityFields []string
}
func cloneInheritancePolicyTemplates(templates []collectionPolicyScope) []collectionPolicyScope {
if templates == nil {
return nil
}
cloned := make([]collectionPolicyScope, len(templates))
for index, template := range templates {
cloned[index] = cloneCollectionPolicyScopeForInheritance(template)
}
return cloned
}
func cloneProjectedInheritancePolicies(policies []projectedCollectionPolicy) []projectedCollectionPolicy {
if policies == nil {
return nil
}
cloned := make([]projectedCollectionPolicy, len(policies))
for index, policy := range policies {
cloned[index] = projectedCollectionPolicy{
path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
}
}
return cloned
}
func cloneInheritanceTransfer(transfer inheritanceTransfer) inheritanceTransfer {
cloned := transfer
cloned.from = transfer.from.clone()
cloned.to = transfer.to.clone()
cloned.reference = transfer.reference.clone()
cloned.locations = append([]Location(nil), transfer.locations...)
cloned.policyTemplates = cloneInheritancePolicyTemplates(transfer.policyTemplates)
cloned.templatePolicies = cloneCollectionPolicyScopeForInheritance(transfer.templatePolicies)
cloned.projectedPolicies = cloneProjectedInheritancePolicies(transfer.projectedPolicies)
return cloned
}
func cloneInheritanceOperations(operations []inheritanceOperation) []inheritanceOperation {
if operations == nil {
return nil
}
cloned := make([]inheritanceOperation, len(operations))
for index, operation := range operations {
cloned[index] = operation
if operation.control != nil {
control := *operation.control
control.path = operation.control.path.clone()
control.locations = append([]Location(nil), operation.control.locations...)
cloned[index].control = &control
}
if operation.transfer != nil {
transfer := cloneInheritanceTransfer(*operation.transfer)
cloned[index].transfer = &transfer
}
cloned[index].parent = document.Clone(operation.parent)
cloned[index].effective = document.Clone(operation.effective)
cloned[index].parentOrigins = cloneOriginIndex(operation.parentOrigins)
cloned[index].effectiveOrigins = cloneOriginIndex(operation.effectiveOrigins)
cloned[index].parentPolicies = cloneInheritancePolicyTemplates(operation.parentPolicies)
cloned[index].templatePolicies = cloneCollectionPolicyScopeForInheritance(operation.templatePolicies)
cloned[index].effectivePolicies = cloneInheritancePolicyTemplates(operation.effectivePolicies)
cloned[index].sequences = cloneSequenceHistoryContributions(operation.sequences)
cloned[index].identityFields = cloneStrings(operation.identityFields)
}
return cloned
}
func inheritancePolicyScopeAtPath(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
return newInheritancePolicyIndex(scope).localize(prefix)
}
func projectInheritancePolicyTemplates(templates []collectionPolicyScope, prefix Path) []projectedCollectionPolicy {
var projected []projectedCollectionPolicy
for _, template := range templates {
for _, policy := range projectCollectionPolicyScope(template, prefix) {
projected = append(projected, projectedCollectionPolicy{
path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
})
}
}
return projected
} }
// A replacement retains the displaced source/derivation record while the // A replacement retains the displaced source/derivation record while the
@@ -66,13 +153,6 @@ type inheritanceContributor struct {
trace *inheritanceTrace trace *inheritanceTrace
} }
type siblingFrame struct {
name string
parent string
hasParent bool
entered bool
}
// expandSiblingDefinitions resolves only the mappings directly in scope. // expandSiblingDefinitions resolves only the mappings directly in scope.
// Nested instructions remain in the returned templates for destination-local // Nested instructions remain in the returned templates for destination-local
// resolution. This private entry point does not participate in public loading. // resolution. This private entry point does not participate in public loading.
@@ -84,202 +164,6 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li
return expandSiblingDefinitionsWithOrigins(scope, scopePath, limits, nil) return expandSiblingDefinitionsWithOrigins(scope, scopePath, limits, nil)
} }
func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
if scope == nil || scope.Kind != document.Mapping {
return nil, []Diagnostic{{
Code: CodeInheritanceInvalid, Severity: SeverityError, Path: scopePath,
Message: "inheritance scope must be a mapping",
}}
}
byName := make(map[string]document.Entry, len(scope.Mapping))
names := make([]string, 0, len(scope.Mapping))
for _, entry := range scope.Mapping {
byName[entry.Key] = entry
names = append(names, entry.Key)
}
slices.Sort(names)
resolved := make(map[string]*document.Node, len(byName))
resolvedOrigins := make(map[string]map[string]Origin, len(byName))
state := make(map[string]uint8, len(byName))
result := &inheritanceScope{}
counter := newRetainedNodeCounter(historyNodeLimit)
if len(budget) != 0 {
counter = budget[0]
}
retained := 1 // scope mapping node
for _, entry := range scope.Mapping {
if entry.Value.Kind == document.Mapping {
continue
}
remaining := limits.nodes - retained - 1
if limits.nodes >= 0 && remaining < 0 {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(scopePath.Key(entry.Key), entry.Value.Location)}
}
count, within := inheritanceLowerBoundCount(entry.Value, remaining)
if !within {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(scopePath.Key(entry.Key), entry.Value.Location)}
}
retained += 1 + count // key and value subtree
}
for _, name := range names {
if byName[name].Value.Kind != document.Mapping || state[name] == 2 {
continue
}
stack := []siblingFrame{{name: name}}
for len(stack) != 0 {
frame := &stack[len(stack)-1]
entry := byName[frame.name]
path := scopePath.Key(frame.name)
if !frame.entered {
state[frame.name] = 1
frame.entered = true
reference, found := inheritanceDirective(entry.Value)
if found {
referencePath := path.Key(inheritanceKey)
location := publicLocation(reference.Value.Location)
result.controls = append(result.controls, inheritanceControl{
path: referencePath, locations: []Location{location},
cancelled: reference.Value.Kind == document.Null,
})
switch reference.Value.Kind {
case document.Null:
case document.String:
frame.parent, frame.hasParent = reference.Value.Text, true
parent, exists := byName[frame.parent]
if !exists {
return nil, []Diagnostic{inheritanceDiagnostic(CodeInheritanceMissing, referencePath, location, nil,
"referenced sibling does not exist")}
}
if parent.Value.Kind != document.Mapping {
related := []Location{publicLocation(parent.Value.Location)}
return nil, []Diagnostic{inheritanceDiagnostic(CodeInheritanceInvalid, referencePath, location, related,
"referenced sibling is not a mapping")}
}
if state[frame.parent] == 1 {
related := []Location{publicLocation(parent.Value.Location)}
return nil, []Diagnostic{inheritanceDiagnostic(CodeInheritanceCycle, referencePath, location, related,
"sibling inheritance contains a cycle")}
}
if state[frame.parent] == 0 {
stack = append(stack, siblingFrame{name: frame.parent})
continue
}
default:
return nil, []Diagnostic{inheritanceDiagnostic(CodeInheritanceInvalid, referencePath, location, nil,
"inheritance reference must be a sibling name or null")}
}
}
}
payload := withoutInheritanceDirective(entry.Value)
operationStart, replacementStart := len(result.operations), len(result.replacements)
childOrigins := directInheritanceOrigins(payload, path, inputOrigins)
if reference, found := inheritanceDirective(entry.Value); found {
control := inheritanceControl{
path: path.Key(inheritanceKey),
locations: []Location{publicLocation(reference.Value.Location)},
cancelled: reference.Value.Kind == document.Null,
}
controlOrigins := directInheritanceOrigins(reference.Value, control.path, inputOrigins)
result.operations = append(result.operations, inheritanceOperation{
control: &control, controlOrigin: controlOrigins[control.path.String()],
})
}
if limits.nodes >= 0 {
remaining := limits.nodes - retained - 1 // the sibling key
within := false
if frame.hasParent {
within = inheritanceOverlayLowerBoundWithin(resolved[frame.parent], payload, remaining)
} else {
within = inheritanceLowerBoundWithin(payload, remaining)
}
if !within {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(path, entry.Value.Location)}
}
}
if frame.hasParent {
parent := resolved[frame.parent]
merged := document.Clone(parent)
control := inheritanceDirectiveLocation(entry.Value)
transfer := inheritanceTransfer{
from: scopePath.Key(frame.parent), to: path,
reference: path.Key(inheritanceKey), locations: []Location{control},
}
mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[frame.parent], transfer.from, path, transfer)
mergeInheritanceOrigin(mergedOrigins, path, childOrigins[path.String()])
mergeRecursiveMappings(merged, payload, path, new(mappingIndex), recursiveMergeHooks{
added: func(childPath Path, incoming *document.Node) {
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
},
combined: func(childPath Path, _ *document.Node) {
mergeInheritanceOrigin(mergedOrigins, childPath, childOrigins[childPath.String()])
},
replaced: func(childPath Path, existing, incoming *document.Node) {
previous := displacedOrigins(existing, mergedOrigins, childPath)
result.replacements = append(result.replacements, inheritanceReplacement{
path: childPath.clone(), before: existing, after: incoming,
previous: previous, effective: cloneOrigin(childOrigins[childPath.String()]),
})
deleteOriginSubtree(mergedOrigins, childPath, existing)
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
},
})
merged.Location = entry.Value.Location
resolved[frame.name] = merged
resolvedOrigins[frame.name] = mergedOrigins
result.transfers = append(result.transfers, transfer)
result.operations = append(result.operations, inheritanceOperation{
transfer: &transfer, parent: parent, effective: merged,
parentOrigins: resolvedOrigins[frame.parent], effectiveOrigins: mergedOrigins,
})
} else {
resolved[frame.name] = document.Clone(payload)
resolvedOrigins[frame.name] = childOrigins
}
count, _ := inheritanceLowerBoundCount(resolved[frame.name], limits.nodes)
retained += 1 + count
for _, operation := range result.operations[operationStart:] {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
return nil, []Diagnostic{inheritanceHistoryBudgetDiagnostic(path, entry.Value.Location)}
}
}
for _, replacement := range result.replacements[replacementStart:] {
if !counter.add(replacement.before) || !counter.add(replacement.after) {
return nil, []Diagnostic{inheritanceHistoryBudgetDiagnostic(path, entry.Value.Location)}
}
}
state[frame.name] = 2
stack = stack[:len(stack)-1]
}
}
root := *scope
root.Mapping = make([]document.Entry, len(scope.Mapping))
rootOrigin := directInheritanceOrigins(&document.Node{Kind: document.Mapping, Location: scope.Location}, scopePath, inputOrigins)
result.origins = rootOrigin
for index, entry := range scope.Mapping {
root.Mapping[index] = entry
if resolved[entry.Key] != nil {
root.Mapping[index].Value = resolved[entry.Key]
for pointer, origin := range resolvedOrigins[entry.Key] {
result.origins[pointer] = origin
}
} else {
root.Mapping[index].Value = document.Clone(entry.Value)
for pointer, origin := range directInheritanceOrigins(entry.Value, scopePath.Key(entry.Key), inputOrigins) {
result.origins[pointer] = origin
}
}
}
result.root = &root
if !inheritanceLowerBoundWithin(result.root, limits.nodes) {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(scopePath, scope.Location)}
}
return result, nil
}
func inheritanceDirective(node *document.Node) (document.Entry, bool) { func inheritanceDirective(node *document.Node) (document.Entry, bool) {
for _, entry := range node.Mapping { for _, entry := range node.Mapping {
if entry.Key == inheritanceKey { if entry.Key == inheritanceKey {

View File

@@ -0,0 +1,309 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func projectInheritanceCollectionPolicies(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
projected := collectionPolicyScope{}
for _, entry := range scope.collections {
policy := cloneCollectionPolicyForInheritance(entry.policy)
stripInheritanceModes(&policy)
projected.set(appendDestination(prefix, entry.destination), policy)
}
return projected
}
func projectCurrentInheritanceCollectionPolicies(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
projected := collectionPolicyScope{}
for _, entry := range scope.collections {
projected.set(appendDestination(prefix, entry.destination), cloneCollectionPolicyForInheritance(entry.policy))
}
return projected
}
func projectInheritancePolicyRoots(target *collectionPolicyScope, scope collectionPolicyScope, prefix Path) {
for _, entry := range scope.collections {
target.set(appendDestination(prefix, entry.destination), cloneCollectionPolicyForInheritance(entry.policy))
}
}
func stripInheritanceModes(policy *collectionPolicy) {
if policy == nil {
return
}
policy.explicitMode = 0
policy.explicitModePresent = false
policy.modeLocation = nil
for owner, nested := range policy.recordPolicies {
for key, entry := range nested.collections {
stripInheritanceModes(&entry.policy)
nested.collections[key] = entry
}
policy.recordPolicies[owner] = nested
}
}
func inheritanceDeclarationsAt(path Path, policy collectionPolicy) []sequenceDeclarationSpec {
if !policy.identityPresent && !policy.explicitModePresent {
return nil
}
return []sequenceDeclarationSpec{{
destination: path.clone(),
identityFields: cloneStrings(policy.identityFields), identityPresent: policy.identityPresent,
identityLocation: cloneLocation(policy.identityLocation),
mode: policy.explicitMode, modePresent: policy.explicitModePresent,
modeLocation: cloneLocation(policy.modeLocation),
}}
}
func inheritanceCollectionResolution(
path Path,
lowerScope, higherScope collectionPolicyScope,
applicationPolicy sequencePolicy,
incoming *document.Node,
) (collectionPolicyResolution, collectionPolicy, collectionPolicy, bool, []Diagnostic) {
lower, hasLower := lowerScope.at(path)
higher, hasHigher := higherScope.at(path)
applicationIdentity, registered := applicationPolicy.identityFieldsAt(path)
if !hasLower && !hasHigher && !registered {
return collectionPolicyResolution{}, collectionPolicy{}, collectionPolicy{}, false, nil
}
lowerForResolution := cloneCollectionPolicy(lower)
// The parent policy is copied to a different sibling destination. Its YAML
// identity follows the inherited collection, but its explicit mode does not.
lowerForResolution.explicitMode = 0
lowerForResolution.explicitModePresent = false
lowerForResolution.modeLocation = nil
if incoming != nil && higher.explicitModePresent && higher.explicitMode == sequenceModeReplace {
// Actual replacement discards the parent's YAML identity before
// agreement. A mode-only declaration still governs retained records.
lowerForResolution = collectionPolicy{}
}
var lowerPointer *collectionPolicy
if hasLower {
lowerPointer = &lowerForResolution
}
resolution, diagnostics := resolveCollectionPolicy(
path, inheritanceDeclarationsAt(path, higher), applicationIdentity, lowerPointer,
)
return resolution, lower, higher, true, diagnostics
}
func mergeInheritanceCollectionPolicies(
parent, child collectionPolicyScope,
parentRootPath Path,
parentValue, childValue *document.Node,
applicationPolicy sequencePolicy,
replaced collectionReplacementPaths,
) (collectionPolicyScope, []Diagnostic) {
result := cloneCollectionPolicyScopeForInheritance(parent)
paths := make(map[string]Path, len(parent.collections)+len(child.collections))
for key, entry := range parent.collections {
paths[key] = entry.destination.clone()
}
for key, entry := range child.collections {
paths[key] = entry.destination.clone()
}
var lookup *documentLookup
if len(paths) != 0 {
lookup = buildDocumentLookup(parentValue, childValue)
}
var diagnostics []Diagnostic
for _, path := range paths {
relative, inside := relativeHistoryPath(parentRootPath, path)
var retained, incoming *document.Node
if inside {
if !replaced.contains(path, false) {
retained, _ = documentAtPathWithLookup(parentValue, relative, lookup)
}
incoming, _ = documentAtPathWithLookup(childValue, relative, lookup)
}
resolution, lower, _, active, issues := inheritanceCollectionResolution(path, parent, child, applicationPolicy, incoming)
if len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
continue
}
if !active {
continue
}
var lowerPointer *collectionPolicy
if _, found := parent.at(path); found {
lower.explicitMode = 0
lower.explicitModePresent = false
lower.modeLocation = nil
lowerPointer = &lower
}
policy := resolution.policy
if incoming == nil || resolution.operation != sequenceModeReplace {
var foundDiagnostics []Diagnostic
policy, foundDiagnostics = collectionPolicyWithoutValue(lowerPointer, resolution.policy, retained, path)
if len(foundDiagnostics) != 0 {
diagnostics = append(diagnostics, foundDiagnostics...)
continue
}
}
if higher, ok := child.at(path); ok {
policy = inheritanceRecordPoliciesFromHigher(policy, higher)
}
result.set(path, policy)
}
return result, canonicalDiagnostics(diagnostics)
}
func inheritanceRecordPoliciesFromHigher(policy, higher collectionPolicy) collectionPolicy {
if len(higher.recordOrder) == 0 {
return policy
}
policy.recordOrder = nil
policy.recordPolicies = nil
for index := range higher.recordOrder {
if record, found := higher.recordPolicyAt(index); found {
policy.setRecordPolicyAt(index, currentCollectionRecordPolicy(record))
}
}
return policy
}
func inheritedCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
cloned := cloneCollectionPolicyForInheritance(collectionPolicy{
recordOrder: []*collectionRecordOwner{record.owner},
recordPolicies: map[*collectionRecordOwner]collectionPolicyScope{record.owner: record.scope},
})
if len(cloned.recordOrder) == 0 {
return newCollectionRecordPolicy()
}
owner := cloned.recordOrder[0]
scope := cloned.recordPolicies[owner]
for key, entry := range scope.collections {
stripInheritanceModes(&entry.policy)
scope.collections[key] = entry
}
return collectionRecordPolicy{owner: owner, scope: scope}
}
func currentCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
cloned := cloneCollectionPolicyForInheritance(collectionPolicy{
recordOrder: []*collectionRecordOwner{record.owner},
recordPolicies: map[*collectionRecordOwner]collectionPolicyScope{record.owner: record.scope},
})
if len(cloned.recordOrder) == 0 {
return newCollectionRecordPolicy()
}
owner := cloned.recordOrder[0]
return collectionRecordPolicy{owner: owner, scope: cloned.recordPolicies[owner]}
}
func applyInheritanceSequencePolicy(
path Path,
combined sequenceCombinationResult,
resolution collectionPolicyResolution,
lowerScope, higherScope collectionPolicyScope,
) collectionPolicy {
policy := cloneCollectionPolicyForInheritance(resolution.policy)
policy.recordOrder = nil
policy.recordPolicies = nil
if combined.value == nil || combined.value.Kind != document.Sequence {
return policy
}
lowerPolicy, hasLower := lowerScope.at(path)
higherPolicy, hasHigher := higherScope.at(path)
lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems)
higherProjection := newSequenceItemProjectionIndex(combined.higherItems)
for outputIndex := range combined.value.Sequence {
if higherIndex, exists := higherProjection.inputForOutput(outputIndex); exists {
if record, found := higherPolicy.recordPolicyAt(higherIndex); hasHigher && found {
policy.setRecordPolicyAt(outputIndex, currentCollectionRecordPolicy(record))
} else {
policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
}
continue
}
if lowerIndex, exists := lowerProjection.inputForOutput(outputIndex); exists && hasLower {
if record, found := lowerPolicy.recordPolicyAt(lowerIndex); found {
policy.setRecordPolicyAt(outputIndex, inheritedCollectionRecordPolicy(record))
}
}
}
return policy
}
func applyInheritanceSequenceOrigins(
result *inheritanceScope,
origins, incomingOrigins map[string]Origin,
path Path,
lower, higher *document.Node,
combined sequenceCombinationResult,
operation sequenceMode,
) {
if operation == sequenceModeReplace {
if lower != nil {
displaced := displacedOrigins(lower, origins, path)
result.replacements = append(result.replacements, inheritanceReplacement{
path: path.clone(), before: lower, after: higher,
previous: displaced, effective: cloneOrigin(incomingOrigins[path.String()]),
})
deleteOriginSubtree(origins, path, lower)
}
mergeInheritanceOrigin(origins, path, incomingOrigins[path.String()])
if higher != nil && higher.Kind != document.Sequence {
copyInheritanceSubtree(origins, incomingOrigins, higher, path)
}
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
return
}
mergeInheritanceOrigin(origins, path, incomingOrigins[path.String()])
for _, replacement := range combined.replacements {
if lower == nil || lower.Kind != document.Sequence || replacement.lowerIndex < 0 || replacement.lowerIndex >= len(lower.Sequence) ||
higher == nil || higher.Kind != document.Sequence || replacement.higherIndex < 0 || replacement.higherIndex >= len(higher.Sequence) {
continue
}
lowerPath := path.Index(replacement.lowerIndex)
higherPath := path.Index(replacement.higherIndex)
outputPath := path.Index(replacement.outputIndex)
previous := displacedOrigins(lower.Sequence[replacement.lowerIndex], origins, lowerPath)
deleteOriginSubtree(origins, lowerPath, lower.Sequence[replacement.lowerIndex])
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[replacement.higherIndex], higherPath, outputPath)
result.replacements = append(result.replacements, inheritanceReplacement{
path: outputPath, before: lower.Sequence[replacement.lowerIndex], after: higher.Sequence[replacement.higherIndex],
previous: previous, effective: cloneOrigin(incomingOrigins[higherPath.String()]),
})
}
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
}
func validateRegisteredInheritanceSequences(root *document.Node, rootPath Path, applicationPolicy sequencePolicy) []Diagnostic {
var diagnostics []Diagnostic
var visit func(*document.Node, Path)
visit = func(node *document.Node, path Path) {
if node == nil {
return
}
if node.Kind == document.Sequence {
if fields, registered := applicationPolicy.identityFieldsAt(path); registered {
if _, issues := validateSequenceIdentity(node, path, fields, sequenceCombinationIdentityCodes); len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
}
}
for index, item := range node.Sequence {
visit(item, path.Index(index))
}
return
}
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
visit(entry.Value, path.Key(entry.Key))
}
}
}
visit(root, rootPath)
return canonicalDiagnostics(diagnostics)
}

View File

@@ -0,0 +1,369 @@
package engine
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestInheritanceReplacementResetsOnlyReplacedYamlIdentity(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name, key, value, applicationKey string
mode sequenceMode
wantFields []string
conflict bool
}{
{name: "changed key", key: "name", value: "[{name: child}]", mode: sequenceModeReplace, wantFields: []string{"name"}},
{name: "unkeyed replacement", value: "[{name: child}]", mode: sequenceModeReplace},
{name: "non-sequence replacement", value: "42", mode: sequenceModeReplace},
{name: "agreeing application", key: "name", value: "[{name: child}]", applicationKey: "name", mode: sequenceModeReplace, wantFields: []string{"name"}},
{name: "conflicting application", key: "name", value: "[{name: child}]", applicationKey: "id", mode: sequenceModeReplace, conflict: true},
{name: "application still validates replacement", value: "[{name: child}]", applicationKey: "id", mode: sequenceModeReplace, conflict: true},
{name: "mode-only key change", key: "name", mode: sequenceModeReplace, conflict: true},
{name: "mode-only keeps identity", mode: sequenceModeReplace, wantFields: []string{"id"}},
{name: "append key change", key: "name", value: "[{name: child}]", mode: sequenceModeAppend, conflict: true},
{name: "upsert key change", key: "name", value: "[{name: child}]", mode: sequenceModeUpsert, conflict: true},
} {
t.Run(test.name, func(t *testing.T) {
input := "base: {items: [{id: parent}]}\nchild:\n extends: base\n"
if test.value != "" {
input += " items: " + test.value + "\n"
}
root := parseItemPolicyDocument(t, "replacement", input)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
higher := collectionPolicy{explicitMode: test.mode, explicitModePresent: true}
if test.key != "" {
higher.identityFields, higher.identityPresent = []string{test.key}, true
}
policies.set(Keys("child", "items"), higher)
application := sequencePolicy{}
if test.applicationKey != "" {
application = mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("child"), literalSequenceSelector("items")}, identityFields: []string{test.applicationKey},
}})
}
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, application, defaultProjectionLimits())
if test.conflict {
wantCode := CodeSequenceConflict
if test.applicationKey != "" && test.key == "" {
wantCode = CodeSequenceInvalid
}
if len(diagnostics) == 0 || diagnostics[0].Code != wantCode || diagnostics[0].Path.String() != "/child/items" && diagnostics[0].Path.String() != "/child/items/0/id" {
t.Fatalf("missing identity/constraint failure: %#v", diagnostics)
}
return
}
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, found := expanded.collectionPolicies.at(Keys("child", "items"))
if !found || !reflect.DeepEqual(policy.identityFields, test.wantFields) || policy.identityPresent != (test.wantFields != nil) {
t.Fatalf("effective YAML identity = %#v, want %v", policy, test.wantFields)
}
if test.value != "" {
actual, _ := documentAtPath(expanded.root, Keys("child", "items"))
want, _ := documentAtPath(root, Keys("child", "items"))
if !configurationValuesEqual(actual, want) {
t.Fatal("explicit replacement retained parent payload")
}
}
})
}
}
func TestLoadInheritanceReplacementUsesNewYamlKey(t *testing.T) {
t.Parallel()
loaded, err := testLoad[map[string]any](Bytes("replacement", []byte(`configkit:
sequences:
- {destination: [base, items], key: id}
- {destination: [child, items], key: name, mode: replace}
base: {items: [{id: parent}]}
child:
extends: base
items: [{name: child}]
`)))
if err != nil {
t.Fatal(err)
}
value, found := loaded.Explain(Keys("child", "items"))
if !found || value.Effective.YAML != "- name: child\n" {
t.Fatalf("replacement payload = %#v", value)
}
}
func TestInheritanceCombinesIdentifiedSequencesAtDestination(t *testing.T) {
root := parseItemPolicyDocument(t, "inheritance-values", `base:
items:
- id: shared
value: parent
child:
extends: base
items:
- id: shared
value: child
`)
modeLocation := Location{SourceID: "base-policy", Line: 4, Column: 5}
policy := collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true, modeLocation: &modeLocation,
}
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), policy)
expanded, diagnostics := expandInheritanceWithCollectionRules(
root, nil, policies, sequencePolicy{}, defaultProjectionLimits(),
)
if len(diagnostics) != 0 {
t.Fatalf("expand identified inheritance: %#v", diagnostics)
}
items, found := documentAtPath(expanded.root, Keys("child", "items"))
if !found || items.Kind != document.Sequence || len(items.Sequence) != 1 {
t.Fatalf("child items = %#v, found %t; parent mode should not force an append", items, found)
}
if value, exists := documentAtPath(items.Sequence[0], Keys("value")); !exists || value.Text != "child" {
t.Fatalf("matched record = %#v, want complete child replacement", items.Sequence[0])
}
childPolicy, found := expanded.collectionPolicies.at(Keys("child", "items"))
if !found || !childPolicy.identityPresent || childPolicy.identityFields[0] != "id" {
t.Fatalf("inherited destination identity = %#v, found %t; scope=%#v", childPolicy, found, expanded.collectionPolicies.collections)
}
if childPolicy.explicitModePresent {
t.Fatalf("parent explicit mode leaked to a different destination: %#v", childPolicy)
}
var transfer *inheritanceOperation
for index := range expanded.operations {
if expanded.operations[index].transfer != nil && expanded.operations[index].transfer.to.String() == "/child" {
transfer = &expanded.operations[index]
break
}
}
if transfer == nil || len(transfer.sequences) != 1 || transfer.sequences[0].path.String() != "/child/items" || len(transfer.sequences[0].replacements) != 1 {
t.Fatalf("inheritance sequence projection = %#v, want one mapped replacement", transfer)
}
if origin, exists := expanded.origins["/child/items/0/value"]; !exists || len(origin.Locations) == 0 || origin.Locations[0].SourceID != "inheritance-values" {
t.Fatalf("replacement origin lost the child's source context: %#v", origin)
}
}
func TestLoadAppliesYamlCollectionIdentityDuringSiblingInheritance(t *testing.T) {
input := `configkit:
sequences:
- destination: [base, items]
key: id
mode: append
base:
items:
- id: shared
value: parent
child:
extends: base
items:
- id: shared
value: child
`
loaded, err := Load[map[string]any](Bytes("inheritance-policy", []byte(input)), testSecretBinding())
if err != nil {
if loadError, ok := err.(*LoadError); ok {
t.Fatalf("load inherited collection policy: %v; diagnostics=%#v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
value, found := loaded.Inspection.Explain(Keys("child", "items").Index(0).Key("value"))
if !found || value.Effective.YAML != "child\n" {
t.Fatalf("inherited sequence value = %#v, found %t", value, found)
}
if len(value.History) == 0 {
t.Fatal("inherited sequence replacement has no inspection history")
}
parentValueVisible, childValueVisible := false, false
for _, change := range value.History {
parentValueVisible = parentValueVisible || change.Before.YAML == "parent\n" || change.After.YAML == "parent\n"
childValueVisible = childValueVisible || change.After.YAML == "child\n"
}
if !parentValueVisible || !childValueVisible {
t.Fatalf("sequence replacement history omitted its parent or child value: %#v", value.History)
}
}
func TestInheritanceUsesDestinationExplicitModeAndApplicationConstraint(t *testing.T) {
t.Run("destination append", func(t *testing.T) {
root := parseItemPolicyDocument(t, "append", `base:
items:
- id: base
child:
extends: base
items:
- id: child
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
childLocation := Location{SourceID: "child-policy", Line: 4, Column: 5}
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true, modeLocation: &childLocation,
})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand with destination append: %#v", diagnostics)
}
items, _ := documentAtPath(expanded.root, Keys("child", "items"))
if len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "child" {
t.Fatalf("appended items = %#v", items)
}
})
t.Run("new matching namespace", func(t *testing.T) {
root := parseItemPolicyDocument(t, "dynamic", `base:
enabled: true
child:
extends: base
extensions:
- id: duplicate
- id: duplicate
`)
applicationPolicy, diagnostics := compileSequenceIdentityPolicy([]sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{
{kind: sequenceSelectorLiteral, key: "child"},
{kind: sequenceSelectorLiteral, key: "extensions"},
},
identityFields: []string{"id"},
}})
if len(diagnostics) != 0 {
t.Fatalf("compile identity constraint: %#v", diagnostics)
}
_, diagnostics = expandInheritanceWithCollectionRules(root, nil, collectionPolicyScope{}, applicationPolicy, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/child/extensions" {
t.Fatalf("dynamic application constraint diagnostics = %#v", diagnostics)
}
})
}
func TestInheritanceReplacementModesAndCancellation(t *testing.T) {
for _, test := range []struct {
name string
value string
kind document.Kind
}{
{name: "empty sequence", value: "[]", kind: document.Sequence},
{name: "null", value: "null", kind: document.Null},
} {
t.Run(test.name, func(t *testing.T) {
root := parseItemPolicyDocument(t, "replacement", "base:\n items:\n - id: prior\nchild:\n extends: base\n items: "+test.value+"\n")
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeReplace, explicitModePresent: true,
})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand replacement: %#v", diagnostics)
}
items, found := documentAtPath(expanded.root, Keys("child", "items"))
if !found || items.Kind != test.kind || (items.Kind == document.Sequence && len(items.Sequence) != 0) {
t.Fatalf("replacement value = %#v, found %t", items, found)
}
})
}
for _, value := range []string{"null", "{}"} {
t.Run("append rejects "+value, func(t *testing.T) {
root := parseItemPolicyDocument(t, "append-kind", "base:\n items: [{id: prior}]\nchild:\n extends: base\n items: "+value+"\n")
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
})
_, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/child/items" {
t.Fatalf("append kind diagnostics = %#v", diagnostics)
}
})
}
t.Run("incompatible inherited identity", func(t *testing.T) {
root := parseItemPolicyDocument(t, "identity", `base:
items: [{id: one}]
child:
extends: base
items: [{id: two}]
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{identityFields: []string{"slug"}, identityPresent: true})
_, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/child/items" {
t.Fatalf("identity conflict diagnostics = %#v", diagnostics)
}
})
t.Run("cancelled relationship", func(t *testing.T) {
root := parseItemPolicyDocument(t, "cancel", `base:
items: [{id: parent}]
child:
extends: null
items: [{id: local}]
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand cancelled relationship: %#v", diagnostics)
}
if _, found := expanded.collectionPolicies.at(Keys("child", "items")); found {
t.Fatal("cancelled relationship transferred the parent's collection policy")
}
})
}
func TestInheritanceReplacementRetiresRecordLocalCollectionPolicy(t *testing.T) {
root := parseItemPolicyDocument(t, "record-scope", `base:
items:
- id: shared
children:
- key: old
child:
extends: base
items:
- id: shared
children:
- key: new
`)
nested := collectionPolicy{identityFields: []string{"key"}, identityPresent: true, explicitMode: sequenceModeAppend, explicitModePresent: true}
recordScope := collectionPolicyScope{}
recordScope.set(Keys("children"), nested)
outer := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
outer.setRecordPolicyAt(0, newCollectionRecordPolicy())
owner := outer.recordOrder[0]
outer.recordPolicies[owner] = recordScope
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), outer)
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand replacement record: %#v", diagnostics)
}
children, found := documentAtPath(expanded.root, Keys("child", "items").Index(0).Key("children"))
if !found || children.Kind != document.Sequence || len(children.Sequence) != 1 {
t.Fatalf("replacement children = %#v, found %t", children, found)
}
if key, exists := documentAtPath(children.Sequence[0], Keys("key")); !exists || key.Text != "new" {
t.Fatalf("replacement child = %#v, want only new record", children.Sequence[0])
}
childPolicy, exists := expanded.collectionPolicies.at(Keys("child", "items"))
if !exists {
t.Fatal("inherited outer collection policy is missing")
}
record, found := childPolicy.recordPolicyAt(0)
if !found {
t.Fatal("replacement item has no independent record owner")
}
if nestedPolicy, exists := record.scope.at(Keys("children")); exists && nestedPolicy.explicitModePresent {
t.Fatalf("replaced record retained its former nested operation: %#v", nestedPolicy)
}
}

View File

@@ -25,6 +25,12 @@ func appendInheritanceExplanation(explanation *Explanation, working **document.N
} }
before := valueSnapshot(*working, query, secrets) before := valueSnapshot(*working, query, secrets)
replaceHistoryPath(working, operation.transfer.to, operation.parent) replaceHistoryPath(working, operation.transfer.to, operation.parent)
// A record keeps its own identity throughout inheritance. Restore it
// before taking enclosing snapshots as well as direct field snapshots.
for _, field := range operation.identityFields {
identity, _ := documentAtPath(operation.effective, Keys(field))
replaceHistoryPath(working, operation.transfer.to.Key(field), identity)
}
afterParent := valueSnapshot(*working, query, secrets) afterParent := valueSnapshot(*working, query, secrets)
parent, parentPresent := transferredOperationOrigin(operation, operationPathForQuery(operation.transfer.to, query)) parent, parentPresent := transferredOperationOrigin(operation, operationPathForQuery(operation.transfer.to, query))
if pathsIntersect(operation.transfer.to, query) && parentPresent { if pathsIntersect(operation.transfer.to, query) && parentPresent {

View File

@@ -1,6 +1,7 @@
package engine package engine
import ( import (
"slices"
"sort" "sort"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document" "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -73,13 +74,13 @@ func applyInheritanceMetadata(metadata *Metadata, raw *document.Node, rawOrigins
}) })
applyOverlayMetadata(&sourceView, overlays) applyOverlayMetadata(&sourceView, overlays)
metadata.history = sourceView.history metadata.history = sourceView.history
metadata.inheritance = append([]inheritanceOperation(nil), expansion.operations...) metadata.inheritance = cloneInheritanceOperations(expansion.operations)
} }
func historyWithInheritance(path Path, stored []Change, operations []inheritanceOperation) []Change { func historyWithInheritance(path Path, stored []Change, operations []inheritanceOperation) []Change {
split := len(stored) split := len(stored)
for index, change := range stored { for index, change := range stored {
if change.Origin.Layer == LayerResolution { if change.Origin.Layer == LayerCLI || change.Origin.Layer == LayerResolution {
split = index split = index
break break
} }
@@ -114,7 +115,7 @@ func projectedInheritanceChanges(query Path, operation inheritanceOperation) []C
return []Change{{Path: operation.control.path.clone(), Origin: origin, return []Change{{Path: operation.control.path.clone(), Origin: origin,
Dependencies: []Path{operation.control.path.clone()}}} Dependencies: []Path{operation.control.path.clone()}}}
} }
if operation.transfer == nil || !pathsIntersect(operation.transfer.to, query) { if operation.transfer == nil || !pathsIntersect(operation.transfer.to, query) || inheritanceOperationIdentity(operation, query) {
return nil return nil
} }
path := query path := query
@@ -155,6 +156,9 @@ func projectedInheritanceChanges(query Path, operation inheritanceOperation) []C
} }
func transferredOperationOrigin(operation inheritanceOperation, destination Path) (Origin, bool) { func transferredOperationOrigin(operation inheritanceOperation, destination Path) (Origin, bool) {
if inheritanceOperationIdentity(operation, destination) {
return Origin{}, false
}
relative, ok := relativeHistoryPath(operation.transfer.to, destination) relative, ok := relativeHistoryPath(operation.transfer.to, destination)
if !ok { if !ok {
return Origin{}, false return Origin{}, false
@@ -168,3 +172,12 @@ func transferredOperationOrigin(operation inheritanceOperation, destination Path
operation.transfer.from, operation.transfer.to, *operation.transfer) operation.transfer.from, operation.transfer.to, *operation.transfer)
return transferred[destination.String()], true return transferred[destination.String()], true
} }
func inheritanceOperationIdentity(operation inheritanceOperation, path Path) bool {
if operation.transfer == nil || len(operation.identityFields) == 0 {
return false
}
relative, within := relativeHistoryPath(operation.transfer.to, path)
return within && len(relative.segments) != 0 && relative.segments[0].kind != pathSegmentIndex &&
slices.Contains(operation.identityFields, relative.segments[0].key)
}

View File

@@ -126,7 +126,7 @@ func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocume
applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded) applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded)
finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root) finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root)
return &Inspection{ return &Inspection{
metadata: metadata, baseline: baseline, input: expanded.root, metadata: metadata, baseline: baseline, input: expanded.root, preCLI: expanded.root,
inputOrigins: expanded.origins, final: expanded.root, inputOrigins: expanded.origins, final: expanded.root,
applications: composed.applications, inheritance: expanded.operations, applications: composed.applications, inheritance: expanded.operations,
}, expanded }, expanded

Some files were not shown because too many files have changed in this diff Show More