89 Commits

Author SHA1 Message Date
5959537d1a Prepare the ConfigKit v0.10.0 release 2026-10-24 19:45:33 +00:00
0369111d30 Resolve Windows rooted search paths with Go 2026-10-24 19:24:30 +00:00
2fedb0307f Use Go rules for Windows filename components 2026-10-24 19:20:54 +00:00
087050fa25 Isolate user configuration discovery tests by platform 2026-10-24 19:18:13 +00:00
1ac1f8bc9e Preserve failing search candidate diagnostics 2026-10-24 19:16:50 +00:00
4dc3e70877 Validate search filesystem context after selection 2026-10-24 19:13:35 +00:00
f64497cb13 Record discovery algorithm audit findings 2026-10-24 19:03:04 +00:00
68cf5c489b Record configuration discovery review findings 2026-10-24 17:52:20 +00:00
e50fee2ca5 Complete the configuration discovery implementation plan 2026-10-24 16:26:09 +00:00
eec214944c Align discovery ownership and contributor documentation 2026-10-24 16:24:57 +00:00
4f2c113237 Extend discovery coverage across consumer workflows 2026-10-24 16:20:53 +00:00
2d93a5c703 Expose configuration search through the public facade 2026-10-24 16:16:07 +00:00
2f6c1eca00 Integrate configuration searches into engine loading 2026-10-24 16:11:03 +00:00
8642337966 Load ordinary overlays without a main source 2026-10-24 15:59:39 +00:00
53ce6c63d8 Allow absent inputs to use import contexts 2026-10-24 15:55:10 +00:00
2f0900535d Select configuration files through host rules 2026-10-24 15:46:36 +00:00
4748313105 Expand ordered configuration search candidates 2026-10-24 15:43:10 +00:00
e2634c8815 Define immutable configuration searches 2026-10-24 15:39:59 +00:00
424c89780e Plan configuration discovery implementation 2026-10-24 15:04:02 +00:00
92309ddcc3 Update the discovery roadmap baseline 2026-10-24 14:54:56 +00:00
7ff204bf3d Clarify v0.9.0 import and collection migration guidance 2026-10-24 14:48:21 +00:00
b3ec860fac Document the ConfigKit v0.9.0 release 2026-10-24 14:44:58 +00:00
435dea102f Retire the completed remediation record 2026-10-24 14:41:56 +00:00
ea46001f78 Reuse document indexes for collection policy checks 2026-10-24 14:17:09 +00:00
a0e5693d07 Share indexed fallback locations during effective projection 2026-10-24 14:13:52 +00:00
5c5f2af6cc Limit inheritance policy work to the relevant scope 2026-10-24 14:10:44 +00:00
8253993005 Delete CLI metadata through the replaced subtree 2026-10-24 14:05:42 +00:00
d0848290da Keep sequence projection storage proportional to assigned items 2026-10-24 14:04:26 +00:00
e94a0642ff Keep global registrations out of item-local composition 2026-10-24 13:59:34 +00:00
571d063675 Retire the completed sequence roadmap and implementation plan 2026-10-24 13:56:19 +00:00
d0a605afbb Document the internal complexity refactor roadmap 2026-10-24 02:02:43 +00:00
7a83704f43 Clarify discovery contexts and filename selection boundaries 2026-10-24 02:00:17 +00:00
ac5bf29ddc Settle discovery slug and custom path rules 2026-10-24 01:58:48 +00:00
cce8356b8e Settle empty discovery searches and filename selections 2026-10-24 01:55:02 +00:00
b746dcc609 Settle discovery behavior when no main file is found 2026-10-24 01:51:08 +00:00
9c32b8d3b5 Record sequence algorithm audit findings 2026-10-24 01:46:55 +00:00
c73daf891d Settle discovery directory resolution errors 2026-10-24 01:39:25 +00:00
ae038a5165 Settle discovery presets and directory helpers 2026-10-24 01:33:53 +00:00
367bbaaa1b Clarify sequence operation history in the configuration contract 2026-10-24 01:21:11 +00:00
fafc28421f Build item policies without repeated accumulated copies 2026-10-24 01:20:12 +00:00
1a673753b9 Preserve child identity in inheritance history 2026-10-24 01:17:00 +00:00
988e49bb09 Define discovery filename groups and search presets 2026-10-24 01:16:42 +00:00
5bb4f364b7 Track the main version through sequence movement 2026-10-24 01:11:28 +00:00
af84107905 Document composable configuration discovery 2026-10-24 01:00:08 +00:00
f94dd9b961 Order CLI metadata history after inheritance 2026-10-24 00:55:21 +00:00
146e7e56d9 Retire obsolete collection identity before replacement agreement 2026-10-24 00:51:10 +00:00
d2e20a304a Record sequence feature integration audit findings 2026-10-24 00:38:04 +00:00
8d40983271 Avoid repeated prefix scans when projecting item policies 2026-10-24 00:19:14 +00:00
25cf7ce05c Clarify final identity validation and overlay semantics 2026-10-24 00:16:53 +00:00
57edb8ec19 Identify conflicting sequence registration positions 2026-10-24 00:15:23 +00:00
42663cf07c Retain main version warnings in layered loads 2026-10-24 00:13:54 +00:00
085b57eeb6 Validate final identity fields as application payload 2026-10-24 00:12:31 +00:00
dcf0379e02 Apply nested collection operations when copying mappings 2026-10-24 00:11:21 +00:00
8ecfedc91b Reset inherited YAML identity on actual replacement 2026-10-24 00:07:22 +00:00
f7624ae92a Project incoming source destinations through tier operations 2026-10-24 00:05:24 +00:00
a035df9845 Preserve tier sequence operations in source history 2026-10-24 00:01:46 +00:00
e6598810e0 Project sequence origins from incoming to final item paths 2026-10-23 23:51:33 +00:00
f8cbb2ed87 Preserve retained input presence in nested CLI overrides 2026-10-23 23:47:21 +00:00
af80ccb90c Record sequence implementation review findings 2026-10-23 23:30:27 +00:00
0be7cdb00e Audit sequence identity scaling and lifecycle 2026-10-23 22:03:16 +00:00
e01aeca761 Reconcile sequence identity documentation 2026-10-23 21:57:35 +00:00
d4133f23f5 Add integrated sequence catalog examples 2026-10-23 21:50:10 +00:00
5af71a7533 Validate final registered sequence identities 2026-10-23 21:43:54 +00:00
41b5dff69b Apply CLI overrides after typed decoding 2026-10-23 21:37:13 +00:00
ecfb78d84d Prepare post-decode CLI inspection context 2026-10-23 21:20:19 +00:00
041c4cff02 Apply CLI overrides to decoded configuration 2026-10-23 21:11:54 +00:00
23f22d8982 Prepare reusable CLI override batches 2026-10-23 21:02:46 +00:00
0023617a07 Expose sequence identity registrations through the facade 2026-10-23 20:59:23 +00:00
33a51606da Resolve inheritance between identified records 2026-10-23 20:48:46 +00:00
268fd03579 Prepare reusable inheritance scope execution 2026-10-23 20:32:10 +00:00
9d4d2e2a17 Build identified record inheritance graphs 2026-10-23 20:23:09 +00:00
36ff9170b7 Apply collection policies during sibling inheritance 2026-10-23 20:19:05 +00:00
9e75496664 Prepare collection policy transport through inheritance 2026-10-23 20:04:06 +00:00
80d8c1dc82 Preserve nested collection policy ownership 2026-10-23 19:56:06 +00:00
acd9e9a9fa Integrate collection policies across layered assembly 2026-10-23 19:49:12 +00:00
89a9684f0d Activate selected overlay collection policies 2026-10-23 19:30:48 +00:00
9a7027a04c Activate ordinary source collection declarations 2026-10-23 19:12:39 +00:00
dfeaf90320 Integrate policy-aware item scopes 2026-10-23 18:55:03 +00:00
d03e56398b Resolve collection identity during strict composition 2026-10-23 18:39:04 +00:00
628fc53d2c Separate underlay acquisition from composition 2026-10-23 18:22:37 +00:00
5fec16a768 Separate input acquisition from composition 2026-10-23 18:15:01 +00:00
f6b02dec03 Project sequence evidence through item mappings 2026-10-23 18:07:15 +00:00
7b04dc6a50 Extend history replay for mapped sequence items 2026-10-23 17:55:45 +00:00
12fdf0d331 Implement shared sequence operations 2026-10-23 17:49:05 +00:00
b202dd187f Track collection policy ownership 2026-10-23 14:16:22 +00:00
0314e587b6 Reconcile collection declarations and modes 2026-10-23 14:07:07 +00:00
9d7635d1e4 Parse collection sequence declarations 2026-10-23 14:01:39 +00:00
298e70b552 Add dynamic sequence identity selectors 2026-10-23 13:55:25 +00:00
64c51647f0 Add shared sequence identity primitives 2026-10-23 13:49:22 +00:00
134 changed files with 16092 additions and 3133 deletions

View File

@@ -17,7 +17,8 @@ then report domain errors and warnings through read-only semantic validation. Se
tags and a path predicate drive deterministic redacted effective YAML. Typed secret
references resolve environment names before validation, and explicit
secrets-directory bootstrap is available as a separate startup operation. Explicit
CLI override records apply above overlays. Value explanation is available through
CLI override records apply after inheritance and typed decoding, before application
resolution. Value explanation is available through
detached structured records and deterministic redacted output. Built-in
`extends` relationships let a mapping inherit from a named sibling during the
same load, retaining explicit presence and source/derivation history. The root
@@ -26,12 +27,14 @@ the [configuration contract](docs/config.md#effective-configuration-diffs) and
[consumer guide](docs/consumers/README.md#effective-configuration-diffs) describe
its safe changed-region records and maintained workflows.
Sources can also be explicitly placed as sequence items, and definitions can
register selected sequences to append records with distinct string IDs during
strict composition. Higher-priority sequences still replace lower-priority
sequences as whole values. See the [sequence contract](docs/config.md#strict-document-composition),
[maintained prompt example](docs/consumers/README.md#identified-prompt-sequences),
and [ADR 0014](docs/adr/0014-sequence-placement-and-composition.md).
Sources can also be explicitly placed as sequence items. Configuration
declarations and application registrations can give selected sequences scalar
identities and choose how they combine across precedence and inheritance.
Strict peers combine distinct records, while higher-priority collections can
replace, append, or upsert whole records. See the
[sequence contract](docs/config.md#strict-document-composition),
[maintained catalog examples](docs/consumers/README.md#identified-prompt-sequences),
and [ADR 0017](docs/adr/0017-collection-sequence-policy.md).
Imports are also explicit: a source may name files, search declared directories,
or select files with path globs. Applications can set load-wide import defaults
@@ -42,6 +45,11 @@ ConfigKit does not search conventional application paths implicitly. See the
and [ADR 0015](docs/adr/0015-shared-dependency-ready-ordering.md) and
[ADR 0016](docs/adr/0016-independent-import-controls.md).
Main-file discovery is explicit too: applications choose and order `Search`
descriptions and pass `SearchConfig` to the existing load APIs. See the
[discovery contract](docs/config.md#configuration-discovery), [consumer guide](docs/consumers/README.md#explicit-main-file-discovery),
and [ADR 0019](docs/adr/0019-explicit-configuration-discovery.md).
```go
type Config struct {
Enabled bool `yaml:"enabled"`
@@ -76,12 +84,13 @@ mappings combine, while overlapping non-mapping definitions are errors. Between
precedence tiers, mappings combine recursively, retaining omitted lower-priority
children; other values replace lower-priority values. The main document and its
imports form one strictly composed tier. Explicit underlay groups sit below it
and above typed defaults; a selected overlay and explicit CLI overrides sit
above it. The [configuration contract](docs/config.md#strict-document-composition)
owns the detailed combination rules and input-specific constraints. At
definition-registered paths only, strict composition can append records with
distinct explicit IDs; ordinary and unregistered sequences keep their conflict
or whole-value replacement behavior.
and above typed defaults; a selected overlay applies before inheritance, while
explicit CLI overrides apply after decoding and before application resolution.
The [configuration contract](docs/config.md#strict-document-composition) owns
the detailed combination rules and input-specific constraints. Declared or
registered collections combine distinct identified records within strict tiers
and use their selected operation at precedence boundaries; other sequences keep
their strict-conflict and whole-value-replacement behavior.
The [architecture goals](docs/policy/architecture.md#success-criteria) explain
these priorities and how genuine new needs can justify extending the shared

90
discovery.go Normal file
View File

@@ -0,0 +1,90 @@
package configkit
import "gitea.maximumdirect.net/eric/ConfigKit/internal/engine"
// Search is an immutable ordered description of possible host-file inputs.
// It is consumed only when passed to SearchConfig and used by a load.
type Search struct{ value engine.Search }
// FilenameGroup selects filenames in order for a directory search.
// Construct groups with NamedFilenames, GenericFilenames, or CustomFilenames.
type FilenameGroup struct{ value engine.FilenameGroup }
// SearchOptions controls whether a search must select a file.
type SearchOptions struct {
// Required reports a missing-source error if the complete search is exhausted.
Required bool
}
// DirectorySearchOptions selects the filename groups used by SearchDirectory.
type DirectorySearchOptions struct {
// Filenames concatenates groups in order. A nil slice uses Named then Generic;
// a non-nil empty slice selects no filenames.
Filenames []FilenameGroup
}
// NamedFilenames selects <application>.yml followed by <application>.yaml.
func NamedFilenames() FilenameGroup { return FilenameGroup{value: engine.NamedFilenames()} }
// GenericFilenames selects config.yml, config.yaml, main.yml, and main.yaml.
func GenericFilenames() FilenameGroup { return FilenameGroup{value: engine.GenericFilenames()} }
// CustomFilenames selects literal filename components in the supplied order.
// Names cannot be empty or contain a directory component.
func CustomFilenames(names ...string) FilenameGroup {
return FilenameGroup{value: engine.CustomFilenames(names...)}
}
// SearchDirectory describes a search within directory. Omitted options or a nil
// Filenames slice uses Named then Generic groups. Explicitly empty Filenames
// selects no names and does not resolve the directory base.
func SearchDirectory(directory string, supplied ...DirectorySearchOptions) Search {
options := make([]engine.DirectorySearchOptions, len(supplied))
for index, option := range supplied {
if option.Filenames == nil {
continue
}
options[index].Filenames = make([]engine.FilenameGroup, len(option.Filenames))
for groupIndex, group := range option.Filenames {
options[index].Filenames[groupIndex] = group.value
}
}
return Search{value: engine.SearchDirectory(directory, options...)}
}
// Paths describes literal host-file candidates in the supplied order.
func Paths(paths ...string) Search { return Search{value: engine.Paths(paths...)} }
// Searches concatenates descriptions in order and snapshots their entries.
func Searches(searches ...Search) Search {
values := make([]engine.Search, len(searches))
for index, search := range searches {
values[index] = search.value
}
return Search{value: engine.Searches(values...)}
}
// UnixSystemDirectories describes the conventional Unix system directories.
// Loading on an unsupported platform fails if this preset is reached.
func UnixSystemDirectories() Search { return Search{value: engine.UnixSystemDirectories()} }
// UserConfigDirectory describes searches rooted at the Go user-config directory.
func UserConfigDirectory() Search { return Search{value: engine.UserConfigDirectory()} }
// HomeDirectory describes searches rooted at the Go home directory.
func HomeDirectory() Search { return Search{value: engine.HomeDirectory()} }
// WorkingDirectory describes application-named files in the current directory.
func WorkingDirectory() Search { return Search{value: engine.WorkingDirectory()} }
// SearchConfig creates a source that selects the first matching host file when
// consumed by Load or LoadLayered. Without Required, exhausting the search uses
// the absent-input lifecycle. Description and option errors are reported while
// loading.
func SearchConfig(application string, search Search, supplied ...SearchOptions) Source {
options := make([]engine.SearchOptions, len(supplied))
for index, option := range supplied {
options[index] = engine.SearchOptions{Required: option.Required}
}
return Source{value: engine.SearchConfig(application, search.value, options...)}
}

View File

@@ -138,6 +138,6 @@ Record inheritance also requires parent lookup by identity and checking for
missing parents and cycles. Copied settings retain the same source, history,
and sensitivity information as mapping-sibling inheritance.
The configuration contract will define declaration syntax, identity comparison,
omission and empty-value behavior, and diagnostics when the feature is
implemented.
The [configuration contract](../config.md#strict-document-composition) defines
declaration syntax, identity comparison, omission and empty-value behavior, and
diagnostics for the implemented feature.

View File

@@ -80,5 +80,5 @@ must remain consistent throughout loading.
CLI input can no longer add or cancel inheritance relationships or bypass an
earlier error in source assembly or inheritance. It also cannot fix a value that
fails the earlier decoding step. Semantic validation still runs after overrides
and application resolution. The configuration contract and consumer guidance
must explain these behavior changes when they are implemented.
and application resolution. The [configuration contract](../config.md#explicit-cli-overrides)
and [consumer guidance](../consumers/README.md) describe this behavior.

View File

@@ -0,0 +1,68 @@
# ADR 0019: Keep Configuration Discovery Explicit and Inside Loading
## Status
Accepted.
## Date
2026-10-24
## Context
Applications often need a small ordered convention for choosing a main
configuration file. Keeping that choice entirely in each application repeats
path selection and file-loading plumbing. Moving the convention into ConfigKit
without an explicit declaration would make ordinary loads search paths that a
consumer did not choose, obscuring both precedence and filesystem access.
ConfigKit already owns synchronous source acquisition, parsing, composition,
decoding, validation, and inspection. A selected host file can therefore enter
that lifecycle as the existing file source, preserving its source identity,
resource context, and diagnostics. Optional exhaustion can use the same absent
main lifecycle as an explicitly absent source.
## Decision
Consumers choose and order the main-file conventions they want to use. They may
describe literal paths, directory filename groups, and supported directory
presets, and may mark the complete search as required. Search descriptions are
explicit inputs to loading; no preset or convention is activated implicitly.
During synchronous loading, ConfigKit resolves the description to one real host
file or to absence before parsing. The selected file follows the existing source
acquisition and processing rules. A selected-file access, parse, version,
composition, decode, or validation failure is terminal and does not resume the
search. Optional exhaustion shares the absent-main lifecycle; required
exhaustion reports a missing-source diagnostic. No candidate-attempt history is
added to public metadata.
This keeps convention choice with applications while moving repeated selection
and loading plumbing into the library's established lifecycle. It preserves
explicit control, ordinary source behavior, and complete-on-return ownership.
## Alternatives Considered
- **Keep all search and fallback loading in each application.** This preserves
consumer ownership but repeats the file-selection and load coordination that
ConfigKit already owns, and risks divergent handling of selected-file errors.
- **Search conventional locations implicitly during every load.** This reduces
setup, but activates filesystem access and precedence the consumer did not
select and makes ordinary source behavior less predictable.
- **Return candidate paths for the application to load one by one.** This keeps
selection outside the load lifecycle and encourages repeated complete loads,
potentially rerunning defaults, callbacks, and validation during fallback.
## Consequences
The public search description must be immutable and root-owned, with private
engine conversion and selection. The contract documents candidate ordering,
selection, exhaustion, context, and provenance; the API declaration documents
the public symbols. Dynamic bases and file availability are observed when each
load consumes the description. Applications remain responsible for selecting
conventions and for their schema and domain requirements.
The [configuration contract](../config.md) owns implemented behavior, and the
[component overview](../internal/overview.md) describes implementation ownership.
The completed implementation plan and remediation record are preserved in Git
history at `0369111` under `docs/roadmap/`.

View File

@@ -12,6 +12,7 @@ ConfigKit exposes opaque sources through these constructors:
| Constructor | Current contract |
| --- | --- |
| `File(path)` | Requires a nonempty path, opens a regular file for each consumption, and closes only the library-owned file handle. Missing, inaccessible, unreadable, directory, and other nonregular inputs fail. |
| `SearchConfig(application, search, options...)` | Creates an explicit, reusable search source. Loading selects one host file or follows the absent-input lifecycle when an optional search is exhausted. |
| `NewFilesystem(label, filesystem).File(name)` | Borrows a caller-owned `fs.FS` under a safe nonempty display label. `name` must be a non-root slash-relative `fs.ValidPath`. Each load reopens the selected regular file; copies of the `Filesystem` preserve namespace identity. Separate constructors create separate namespaces even for the same label or filesystem. |
| `Bytes(label, data)` | Requires a nonempty, non-secret diagnostic label and snapshots bytes during construction. Later caller mutation cannot affect the source. |
| `Reader(label, input)` | Requires a nonempty, non-secret label and non-nil reader. The reader is claimed once across all copies of the source, consumed but never closed, and cannot be reused. |
@@ -28,6 +29,79 @@ Keys are sorted for stable provenance, integer decimal spelling stays exact, and
cycles or unsupported types are rejected without exposing values. Mapping input
uses the same directive and version rules as YAML, not CLI override restrictions.
### Configuration discovery
Discovery is opt-in. Construct `Search` from literal `Paths`, directory searches,
or the supported directory presets, then pass it to `SearchConfig`. Search
descriptions are immutable and reusable; each load resolves dynamic directories
and checks current candidate files again. `Searches` is ordered concatenation.
It preserves the supplied description order and does not deduplicate candidates.
`NamedFilenames` selects `<application>.yml`, then `<application>.yaml`.
`GenericFilenames` selects `config.yml`, `config.yaml`, `main.yml`, then
`main.yaml`. `SearchDirectory` defaults to `[Named, Generic]` when options are
omitted or `Filenames` is nil. A non-nil empty filename list contributes no
candidates and does not resolve its directory. `CustomFilenames` supplies
literal single filename components; it does not expand application-name
placeholders. An application slug is required only when a selected directory or
filename group uses it. Literal custom paths are not expanded for `~`, variables,
or wildcards; relative paths are resolved from the working directory at load time.
The zero `Search`, `Searches()`, `Paths()`, and an empty custom filename group
contribute no candidates. An explicitly supplied empty path or filename is
invalid. Slugs and custom filenames must be usable single host filename
components; ConfigKit preserves their case and spelling.
Search expansion is ordered concatenation. For each directory search, ConfigKit
tries its filename groups and names before advancing to the next directory.
When multiple directories use the same filename list, candidates follow the
directory-first Cartesian product: every filename in one directory is tried
before the next directory. Explicit file candidates stay at their declared
positions among directory searches.
The presets expand to these ordered directories and filename groups:
| Preset | Directory, in order | Filename groups |
| --- | --- | --- |
| `UnixSystemDirectories` | `/usr/local/etc/<application>`, `/etc/<application>` | Named, then Generic |
| `UnixSystemDirectories` | `/etc` | Named only |
| `UserConfigDirectory` | Go user-config directory / `<application>` | Named, then Generic |
| `UserConfigDirectory` | Go user-config directory | Named only |
| `HomeDirectory` | Go home directory / `.<application>` | Named, then Generic |
| `HomeDirectory` | Go home directory | Named only |
| `WorkingDirectory` | Current working directory | Named only |
The Unix system preset is supported on aix, android, darwin, dragonfly, freebsd,
hurd, illumos, ios, linux, netbsd, openbsd, and solaris; reaching it on another
platform is an option error. Other presets use Go's user-config, home, and
working-directory helpers. A reached
directory-helper error terminates selection. Bases for later searches are not
resolved after an earlier candidate is selected. Missing candidates continue;
the first existing candidate is selected and then follows ordinary file loading
rules. Errors while acquiring or processing a selected file are terminal and do
not resume the search.
Searches are optional by default. Exhaustion uses the same absent-main lifecycle
as `NoSource()`: there is no main document, source record, source ID, or explicit
presence contribution. Imports may be enabled but there are no main declarations
to traverse. Defaults, version checks, overlays, CLI overrides, resolution, and
validation retain their normal lifecycle. `SearchOptions{Required: true}` makes
exhaustion an error; it does not change candidate or selected-file error handling.
For an ordinary active overlay after optional exhaustion, provide an explicit
filesystem context. A host context with only `Root` is sufficient; ConfigKit
loads the overlay at its normal root-relative location and does not infer a root
from attempted search directories. The same root-only context can accompany a
search that selects a file: the selected file determines its base and must be
inside `Root`. A selected host file rejects an explicit `BaseDir` or native
filesystem context. Invalid context fails without resuming selection. Layered
overlays keep their independent `OverlayFilesystem` requirement.
When a search selects a file, inspection reports that actual file with ordinary
source IDs, paths, placement, metadata origins, and resource context. Missing
candidates create no records. `SearchConfig` can be used as the main source of
ordinary or layered loading, including an underlay input, and does not add a new
precedence tier.
Each source is limited to 8 MiB. A load using imports or an active overlay is limited
to 64 MiB across all accepted sources, 1,024 accepted sources, 64 active import
edges, and 1,000,000 input nodes in aggregate. Glob expansion separately permits
@@ -52,10 +126,11 @@ preserves already validated secret state without rereading the environment.
The current lifecycle is:
```text
validate options → parse the selected source → traverse/compose enabled imports
validate options → parse the selected source → extract source controls and
traverse/strictly compose enabled imports
→ build schema → check the optional application version → apply a selected overlay
→ apply explicit CLI overrides → expand sibling inheritance
→ create and clone defaults → strictly decode
→ expand sibling inheritance
→ create and clone defaults → strictly decode → apply explicit CLI overrides
→ recorded application resolution
→ resolve typed secret references → semantic validation → classify secrets
→ snapshot configuration and metadata
@@ -70,15 +145,21 @@ missing file remains an error; it is never interpreted as no input.
`Options` accepts the application definition, ordinary-source `Destination` and
`PlacementMode`, explicit imports, one selected overlay, filesystem
context, explicit CLI overrides, and shared source-version controls. The definition
may register strict sequence composition paths through
`SequenceCompositions`. Its zero-valued placement mode places the source mapping
may register sequence identities through `Definition.SequenceIdentities`. Its
zero-valued placement mode places the source mapping
at the destination; `sequence-item` explicitly wraps the mapping as one item at a
non-root destination. Unknown modes, invalid destinations, and invalid or
conflicting sequence registrations fail before source acquisition. Overlay and
CLI coordinates remain rooted at the application root. `NoSource` rejects
imports, active overlays, and filesystem context as invalid combinations but
accepts overrides. Input containing a reserved `configkit` mapping key is rejected
while imports are disabled.
CLI coordinates remain rooted at the application root. Ordinary `Load` recognizes
source-root collection declarations whether or not imports are enabled; an
`imports` declaration still requires imports to be enabled. `NoSource` may enable
imports, but it has no main declarations to traverse. It accepts an explicit
filesystem context: a host context requires an absolute existing `Root` and may
omit `BaseDir` to use that root as its internal import base; a native context
requires its namespace, `BaseDir`, and `Root`. Supplied context is validated even
when imports are disabled. An active ordinary overlay requires this explicit
context. `NoSource` accepts overrides and remains absent rather than contributing
an empty document.
Every source, options, schema, version, defaults, decode, resolution, or validation
failure returns a nil result and a `*LoadError`. Successful results contain caller-owned `Config`, detached
@@ -128,26 +209,34 @@ nothing. Underlay groups are listed from lowest to highest priority. Repeating
a file within one group remains an import/repetition error; the same reusable
source may be selected again in another tier.
ConfigKit acquires and composes the main input first, constructs the schema,
and checks the optional application version from the main tier only. Underlay
roots cannot claim that version path (`underlay.version`); imported fragments
retain `import.version` protection. It then acquires and composes underlay
groups in order, recursively combines the groups, and applies the main tier,
one selected overlay, and explicit CLI overrides. Scalar, sequence, and null
definitions replace lower values between tiers; mappings combine recursively.
Syntax and strict-composition errors inside a tier cannot be masked by a
higher-priority tier. Defaults, typed decoding, resolution, secret lookup,
validation, and publication run once on the final assembled input.
ConfigKit acquires the main input first and constructs the schema. It then
acquires and strictly composes underlay groups from lowest to highest priority,
passing each group's accumulated lower collection policy into the next group.
Each group is applied once as a precedence tier. The main tier is then composed
with the accumulated underlay policy, and the optional application version is
checked against that composed main tier only. Underlay roots cannot claim that
version path (`underlay.version`); imported fragments retain `import.version`
protection. ConfigKit then applies the selected overlay and resolves inheritance.
Scalar, sequence, and null definitions replace lower values between tiers unless
a collection policy defines the sequence operation; mappings combine recursively.
After successful defaults and typed decoding, CLI overrides are applied to that
typed configuration before application resolution. Syntax, strict-composition,
inheritance, and decoding errors cannot be masked by a CLI override. Resolution,
secret lookup, validation, and publication run once on the resulting values.
An active layered overlay requires `LayeredOptions.OverlayFilesystem` explicitly,
independent of the main source's namespace. Its name and root-relative lookup
follow the selected-overlay rules below; inactive overlays ignore the context.
The same path in a different filesystem namespace is not a repeated file.
Ordinary `Load` continues to infer its selected-overlay context from its source.
Ordinary `Load` infers overlay context from sources that provide one; with
`NoSource`, it requires the explicit `Options.Filesystem` context.
Source IDs follow acquisition order—main and its depth-first imports, then each
underlay tier's inputs and depth-first imports, then overlay and CLI—even though
history follows precedence order. Source `Layer` describes precedence while
history follows precedence order. Default main-version checking happens after
underlay composition because the main tier inherits the accumulated collection
policy. Enabled per-source version checks retain acquisition order and source
local headers. Source `Layer` describes precedence while
`Kind` describes acquisition. Underlay records use `LayerUnderlay` and a
one-based `UnderlayIndex` giving their original tier position; all other records
have index zero. Explicit empty groups and `NoSource` consume no source ID.
@@ -213,35 +302,55 @@ is recorded in [ADR 0011](adr/0011-input-destinations.md) and
Any input mapping may declare `extends: name` to inherit from the mapping named
`name` among its immediate siblings. Names are exact keys, including the empty
string; lookup does not search ancestor or unrelated scopes. The document root
and a sequence-element mapping have no sibling namespace for their own string
relationship, though mappings nested inside them can have one. An `extends`
string; lookup does not search ancestor or unrelated scopes. A mapping inside a
sequence can also participate in record inheritance when that sequence has an
effective identity declaration from `configkit.sequences` or an application
registration. In that case, `extends` is a mapping containing exactly the
sequence's identity fields, with scalar values; lookup is limited to that final
sequence. The document root and a sequence-element mapping without an effective
identity declaration have no sibling namespace for non-null string
relationships, though mappings nested inside them can have one. An `extends`
value of null cancels a relationship without looking up a parent. Other value
types, missing or non-mapping parents, and cycles fail the load with value-free
diagnostics.
types, incomplete or extra reference fields, missing parents, and cycles fail
the load with value-free diagnostics.
Within each sibling scope, direct mapping definitions resolve by dependency
readiness. A parent is resolved before a child; among currently eligible
definitions, the lexically smallest canonical semantic pointer is selected,
including newly eligible definitions as soon as their parent completes. This
ordering governs inheritance operations and their history, controls, and
transfers. It does not reorder the mappings in the resulting configuration.
Within each mapping sibling scope or identified sequence, direct definitions
resolve by dependency readiness. A parent is resolved before a child; among
currently eligible definitions, the lexically smallest canonical semantic
pointer is selected, including newly eligible definitions as soon as their
parent completes. This ordering governs inheritance operations and their
history, controls, and transfers. It does not reorder mappings or sequence
records in the resulting configuration.
`extends` is reserved structural input across ordinary and layered loads, every
source kind, imports, selected overlays, and explicit CLI overrides. It is
consumed before typed decoding and need not appear in the application's Go
types. Applications previously using `extends` as an ordinary YAML field must
rename that key and its field. No collection registration or keyword setup is
required.
required for mapping-sibling relationships; identified-sequence relationships
require effective identity fields for that collection.
ConfigKit completes strict composition within tiers and precedence assembly
between tiers before resolving relationships. An omitted `extends` retains a
lower-priority relationship; `extends: null` cancels it while preserving the
child's own settings. Superseded references are historical evidence, not active
relationships to validate. Each child recursively overlays its own settings
over its resolved parent: explicit zero, false, empty string, sequence, and
null values replace inherited values, while mappings combine recursively.
The child retains its key. All effective relationships resolve, including
between tiers before resolving relationships. Only records that survive that
assembly participate in identified-sequence lookup: an upserted parent supplies
its final record, a replaced parent is unavailable, and a higher replacement
without `extends` supersedes a lower relationship. An omitted `extends` retains
a lower-priority mapping-sibling relationship; an identified record's effective
directive depends on the record that survives assembly. `extends: null` cancels
a relationship while preserving the child's own settings. Superseded references
are historical evidence, not active relationships to validate. Each child
recursively overlays its own settings over its resolved parent: explicit zero,
false, empty string, and
null values replace inherited values, while mappings combine recursively. For
identified records, the child's identity fields and their origins remain its
own; parent identity fields do not contribute to the payload.
Sequences at collection destinations use the shared collection operation with
the parent as lower input and the child as higher input. The destination's
explicit mode applies; otherwise a copied identity selects upsert and an
unkeyed destination replaces. A parent's explicit mode does not move to a
different destination. Sequences outside collection destinations remain atomic
replacements.
All effective relationships resolve, including
entries the application may never select for execution. Partial parent
definitions undergo normal typed decoding and application-owned validation,
not a special structural exemption.
@@ -423,10 +532,11 @@ the descendant can therefore retain history after `Lookup` stops reporting it as
effective. Equal-valued input is still an explicit base definition, while a derived
value for absent input remains non-explicit.
Appending registered records during strict composition assigns each contribution
its assembled sequence index in history. A later precedence tier that supplies a
sequence records replacement at the sequence path; history does not imply
identity-based merging across tiers.
Strict composition assigns contributed records their assembled sequence positions
in history. Across tiers, history follows the selected
[collection operation](#strict-document-composition), including append and
whole-record upsert. Incoming contributions use their assigned positions, and
replaced payload remains available as historical evidence.
A mapping assembled from the selected file and imports has one composite base origin
whose locations follow [scoped assembly order](#explicit-imports), not source-record
@@ -501,30 +611,33 @@ historical path. Its effective value and each chronological before/after value a
parser values. `Present=false` with empty YAML means absence or deletion; explicit
null is present and renders as `null`. The history starts with the canonical typed
application-default or Go-zero baseline, followed by relevant underlay and main-tier
contributions in precedence order, the selected overlay, supplied CLI records,
and audited net resolution operations. Within each strict tier, replay follows
contributions in precedence order, the selected overlay, and structural inheritance
events. Typed defaulting and decoding reconcile that source history before supplied
CLI records and audited net resolution operations. Within each strict tier, replay follows
[scoped assembly order](#explicit-imports), not source-record acquisition order.
Structural inheritance events appear after source replay
and before typed reconciliation, including consumed or canceled `extends` input
and equal-valued child replacements. Overlapping parent/child resolution records
Inheritance events include consumed or canceled `extends` input and equal-valued
child replacements. Overlapping parent/child resolution records
are rendered
as one net subtree event with combined reasons and dependencies, rather than
pretending that the recorded assignments ran in lexical order. Explicit equal
writes remain visible. When input replay
differs from the final typed input, a `ChangeDecoding` event records the transition
before domain resolution. Its origin retains the decoded path's source/default
attribution; its reason identifies static-default retention and canonical decoding.
This built-in inspection operation is not accepted by `Resolution.Record`.
writes remain visible. When structural replay differs from the pre-CLI typed input,
a `ChangeDecoding` event records retained defaults and canonical decoding. CLI
contributions then retain their raw input representation while effective and
resolution values use the typed projection. This built-in inspection operation is
not accepted by `Resolution.Record`.
Base, overlay, and CLI history preserves the policy-validated input representation,
including a superseded value that could not have decoded into the final Go field.
Baseline, effective, and resolution values use canonical typed projection. A
decoding event bridges replayed input and canonical typed values, so intermediate
null/scalar replacements cannot incorrectly imply permanent default removal. Mapping
queries reconstruct each source or override contribution, while sequences remain
atomic: a descendant history identifies its presence before and after the containing
sequence replacement without implying element-wise merging. Removed paths remain
explainable; unknown and malformed paths return false.
queries reconstruct each source or override contribution. A sequence snapshot
represents the collection at that point in replay, and sequence diffs remain
atomic. Descendant queries describe their presence and values before and after
containing sequence operations. Combination follows the
[collection operation tables](#strict-document-composition); upsert replaces
matching records in full. Removed paths remain explainable; unknown and malformed
paths return false.
Every snapshot applies the same historical tag, predicate, and recorded-dependency
sensitivity closure as effective rendering before YAML is exposed. A redacted scalar
@@ -611,13 +724,15 @@ must resolve to a nonempty value. Secret references work in typed structs, maps,
pointers, slices, and arrays. A dynamic `any` value does not establish secret
semantics and cannot contain `Secret`; applications must use a typed slot.
ConfigKit resolves only the effective names after recorded application resolution
and before semantic validation. Each unique name is read with `os.LookupEnv` once
per load. Validation can inspect the credential through the deliberately named
`Value()` accessor and distinguish success with `IsResolved()`. Later loads read
the environment again, and cloned defaults are rebuilt from names so a previously
resolved result cannot retain a stale credential. `Load` never writes to the
environment.
ConfigKit resolves only the effective names after CLI assignment and recorded
application resolution, and before semantic validation. CLI overrides may supply
or replace a typed secret reference; lookup uses that effective name and retains
its effective source location for diagnostics. Each unique name is read with
`os.LookupEnv` once per load. Validation can inspect the credential through the
deliberately named `Value()` accessor and distinguish success with `IsResolved()`.
Later loads read the environment again, and cloned defaults are rebuilt from names
so a previously resolved result cannot retain a stale credential. `Load` never
writes to the environment.
Credential bytes are held only in the per-load lookup state and the trusted returned
configuration, and are returned only by `Value`. Secret formatting uses a safe
@@ -681,18 +796,19 @@ files when appropriate; ConfigKit stores no implicit global report.
### Recorded application resolution
An optional `Definition.Resolve` callback runs after successful typed decoding. It
receives the working configuration and a per-load `Resolution` context. The context's
An optional `Definition.Resolve` callback runs after successful defaults, typed
decoding, and explicit CLI assignment. It receives the working configuration and a
per-load `Resolution` context. The context's
`IsExplicit` query uses surviving input presence, so explicit false, zero, empty, and
null are never confused with absence or static defaults.
`Resolution.InputInfo(path)` returns detached effective metadata from the fixed
decoded entry state, before application resolution runs. It includes retained static
defaults, preserves explicit null, and uses the same exact semantic path matching as
`Metadata.Lookup`; missing, malformed, and key/index-shape-mismatched paths return
false without diagnostics. Its `Explicit` value agrees with `IsExplicit` for every
found entry path. Resolution changes, additions, removals, and moves do not alter
this entry view.
post-decode entry state, including explicit CLI overrides, before application
resolution runs. It includes retained static defaults, preserves explicit null, and
uses the same exact semantic path matching as `Metadata.Lookup`; missing,
malformed, and key/index-shape-mismatched paths return false without diagnostics.
Its `Explicit` value agrees with `IsExplicit` for every found entry path. Resolution
changes, additions, removals, and moves do not alter this entry view.
`Resolution.Source(id)` returns a detached record for an already acquired source in
the same load: the main source, imports, selected overlay, and supplied CLI records
@@ -767,8 +883,11 @@ live services as a substitute for configuration validation.
## Strict document composition
ConfigKit strictly composes parsed base documents when `Options.Imports` is true.
Application defaults do not participate in this operation.
Ordinary `Load` applies strict document composition to its parsed source scope;
`Options.Imports` adds imported documents as peers in that scope. Application
defaults do not participate. With imports disabled, the single source still has
its controls extracted and any declared or registered collection identity is
validated, but there are no imported peers to combine.
| Existing node | Incoming node | Result |
| --- | --- | --- |
@@ -779,46 +898,139 @@ Application defaults do not participate in this operation.
| Any other pair of present nodes | Any other pair of present nodes | `compose.conflict` |
Every non-mapping overlap is a conflict, even when values are equal, except for
sequence/sequence contributions at a registered sequence-composition path. Nulls
remain atomic conflicts. Unregistered sequences conflict when repeated and are
never joined by index. Disjoint entries under shared mappings compose, making
stable keyed mappings the natural representation for independently contributed
entries.
sequence/sequence contributions at a destination with effective identity from an
application registration, a current YAML `key` declaration, or retained lower
policy. Nulls remain atomic conflicts.
Sequences without effective identity conflict when repeated and are never joined
by index. Disjoint entries under shared mappings compose, making stable keyed
mappings the natural representation for independently contributed entries.
`Definition.SequenceCompositions` registers literal mapping paths and one literal
`IdentityField` for sequences that should compose independently authored records.
Registration paths follow input-destination key rules: parsed numeric tokens are
literal mapping keys, explicit index segments and reserved `configkit` or
mapping-valued `extends` segments are invalid, and the root registration is
harmless and unused. The identity field is one literal string key; neither the
field name nor identity strings are normalized.
At each registered path, every actual sequence contribution in a strict tier is
validated, including a singleton contribution: each record must be a mapping with
a present string identity field. Distinct exact identity strings append in source
and list order; duplicate identities conflict even if the records are otherwise
equal or disjoint. Empty identity strings are valid. Malformed records produce
`compose.invalid`; duplicate identities produce `compose.conflict`, with the
sequence container and source locations identifying a duplicate without exposing
identity values. Missing paths and non-sequence values do not fail by themselves.
Identical registrations coalesce; conflicting registrations at one path and
invalid registration paths produce `options.invalid`. Unused and root registrations
are allowed and have no effect.
`Definition.SequenceIdentities` contains registrations created by
`SequenceIdentityAt` or `SequenceIdentityMatching`. The exact form accepts a
mapping-only `Path`; the matching form accepts literal-key and any-mapping-key
segments created by `SequenceLiteralKey` and `SequenceAnyMappingKey`. A literal
`"*"` key is distinct from the wildcard constructor. Parsed numeric path tokens
remain literal mapping keys; explicit index segments and reserved `configkit` or
mapping-valued `extends` segments are invalid. The root selector is harmless under
the mapping-root schema. Selector segments consume one mapping level and match a
complete destination; an any-mapping-key segment matches exactly one key, not an
arbitrary prefix or suffix. Each registration requires one or more literal,
direct field names; compound field sets are normalized as sets, and field names
are not otherwise normalized. The same field-set and scalar-equality rules apply
to YAML `key` declarations.
Registration changes strict composition only. Each underlay group and the main
tier has independent duplicate-identity state. Precedence assembly is unchanged:
a present higher-priority sequence replaces the lower sequence as one value,
including at a registered path. Overlays, CLI overrides, defaults, inheritance,
and application resolution do not run the registered identity check. The accepted
rationale is recorded in [ADR 0014](adr/0014-sequence-placement-and-composition.md).
The [maintained prompt catalog](consumers/README.md#identified-prompt-sequences)
shows item path and glob imports, helper-supplied identity, and main-tier
replacement of a lower-priority sequence.
At each selected path, records must provide every effective identity field as a
scalar string, number, boolean, or explicit null. Field names form a set, so
order and repetition do not affect compound identity. The type-tagged tuple
preserves category distinctions: strings compare exactly, including empty
strings; numbers, booleans, and null remain distinct categories. Finite numbers
compare by exact mathematical value, so `1`, `1.0`, and `1e0` are equal, as are
`0` and `-0`, without floating-point rounding. Distinct tuples append in source
and list order; duplicate tuples conflict even when records are otherwise equal
or disjoint. Malformed records produce `compose.invalid`; duplicate identities
produce `compose.conflict`, with paths and source locations identifying a
duplicate without exposing identity values. Missing paths and non-sequence values
do not fail by themselves. Equivalent overlapping registrations coalesce.
Incompatible overlaps and invalid registrations produce `options.invalid` before
source acquisition. Unused and root registrations are allowed and have no effect.
Registrations provide identity for strict composition, sequence operations,
inheritance lookup, and final validation. Dynamic selectors apply at concrete
strict-composition, inheritance, and final-result destinations without requiring
the application to enumerate mapping keys. Each strict tier has independent
duplicate-identity state among its peers; layered precedence and inheritance
then apply the collection operation described below.
Final registration checks run in both ordinary and layered loads after effective
secret lookup and before application semantic validation. They examine actual
final typed sequences, including values introduced by defaults, CLI, or recorded
resolution; they do not require path existence or sequence shape. Diagnostics
use concrete collection and item paths and available source locations, without
exposing identity or credential values. The guarantee ends when loading publishes
successfully and does not cover later caller mutations.
YAML-only identity is enforced during structural composition, precedence, and
identified inheritance; it carries no final-result guarantee after typed
defaults, CLI, or application resolution. Applications that need that guarantee
register the identity constraint.
The [maintained catalog examples](consumers/README.md#identified-prompt-sequences)
show strict peers, precedence operations, inheritance, dynamic destinations, and
the distinction between YAML-only declarations and application registrations.
When an enabled source-version mode is selected, declarations are checked in each
source's own coordinates before placement and composition. Accepted non-main
headers are consumed before strict composition and cannot supply a registered
record identity. Final typed decoding and application validation still run on the
completed configuration.
Ordinary sources, layered inputs, and selected overlays may declare collection
behavior under `configkit.sequences` at the document root. Each entry has a
required literal `destination` and may specify `key` as one string or a nonempty
list of strings, plus a `mode` of `replace`, `append`, or `upsert`. These
controls are removed before application decoding. A declaration destination is
relative to that source's placement, so a mounted fragment's declaration is
mounted with its values. Declarations from strict peers must agree on identity
fields and on any explicitly supplied mode; omission expresses no mode
preference. They apply even when the declaring source has no value at that
destination: a key validates an existing retained sequence or remains pending
until records arrive, while a mode-only declaration affects later combinations
without recombining earlier tiers. A key supplies the identity used by `append`
and `upsert` and is checked against any application registration.
Strict peers at an identified destination first combine distinct records in
source and list order. Duplicate identities among peers or within one
contribution fail under every mode. After strict composition, precedence and
inheritance use one operation for the assembled incoming collection:
| Mode | Behavior at source-precedence and inheritance boundaries |
| --- | --- |
| `replace` | Replace the complete lower sequence with the assembled higher sequence. |
| `append` | Retain lower records and append higher records; duplicate identities fail. |
| `upsert` | Replace matching records in full at their lower positions, retain unmatched lower records, and append new identities. |
Upsert replaces the complete matched record; it does not recursively merge
record fields. Append and upsert require identity. With no explicit mode in the
current tier, an explicit lower mode persists; if none exists, effective identity
selects upsert and its absence selects replace. Inferred modes do not persist.
Explicit modes persist at their destination until changed by a higher explicit
mode, independently of YAML identity lifetime. A declaration alone does not
clear or recombine a collection.
The presence and value kind at a precedence or inheritance boundary determine
whether a collection changes:
| Higher contribution | `replace` | `append` / `upsert` |
| --- | --- | --- |
| Absent | Leave the lower value unchanged. | Leave the lower value unchanged. |
| Explicit `[]` | Clear the sequence. | Contribute no records. |
| Explicit null or another non-sequence value | Replace the lower value, subject to application schema. | Error; the operation requires a sequence. |
An absent lower value can be initialized by an incoming sequence under any mode.
A present lower null or other non-sequence value is not absence: append and
upsert cannot combine with it. An empty sequence still requires a valid mode and
identity where applicable. Null used as an identity component inside a record is
distinct from null used as the collection value.
Declarations govern ordinary strict source composition, including loads with
imports disabled, and each strict scope in layered loading. Before composing a
higher tier's peers, ConfigKit makes applicable lower identity and explicit mode
available; after peer composition, the tier applies its resolved operation
against the lower collection. Selected overlays use the same collection policy
after the base source graph is composed. During mapping-sibling inheritance, a
parent's YAML identity can follow its collection to the child destination, while
the parent's explicit operation stays at the parent destination. The child
destination selects its operation; absent one, effective identity implies
upsert and its absence implies replace. Whole-sequence replacement discards
lower YAML identity and uses the replacing scope's key declaration, if any;
without one, the collection is unkeyed unless an application registration still
supplies identity. Replacement discards policies owned by removed records, while
an independently declared explicit mode and application registration remain
applicable. Item-local
policies follow their owning records: retaining a record retains its rules, while
whole-record upsert replacement discards the displaced record's rules. Matching
identity or reusing an index does not transfer them. These policies do not change
CLI behavior: CLI sequence assignments replace whole sequences after decoding.
A root `configkit.imports` declaration still requires `Options.Imports`;
`NoSource` has no declaration scope.
An empty mapping contributes no child definitions. Two empty mappings compose, and
an empty mapping neither clears nor conflicts with another mapping's children.
Shared mapping containers retain all contributing value locations as their composite
@@ -837,13 +1049,21 @@ Every other name resolves only to `<configuration-root>/overlays/<name>.yml`, wi
no extension fallback, case normalization, inheritance between overlay files,
or overlay stack. A selected missing overlay is an error.
With `NoSource`, the main input stays absent and the explicit filesystem context
supplies the overlay root. The overlay is recorded as the first acquired source
when no other source exists. Host contexts may use the root-only form described
under [single-source loading](#single-source-loading); native contexts retain
their complete namespace, base, and root requirements.
The overlay is a separately policy-validated YAML mapping. It is loaded after the
base import graph is composed and its schema version is checked, but before defaults
and typed decoding. Overlay bytes, nodes, and source records count toward the same
aggregate limits as base inputs. Its canonical path must remain within the configured
root and must not duplicate a file already present in the base graph.
Overlay is intentionally distinct from strict composition:
Collection destinations follow the
[collection operation and presence rules](#strict-document-composition) above.
The following table describes ordinary overlay behavior elsewhere:
| Existing lower node | Incoming overlay node | Result |
| --- | --- | --- |
@@ -852,18 +1072,24 @@ Overlay is intentionally distinct from strict composition:
| Mapping | Mapping | Recurse by mapping key |
| Any other pair of present nodes | Any other pair of present nodes | Replace the entire lower node |
Scalars, sequences, and null are atomic. Thus an overlay sequence replaces rather
than extends a base sequence, `[]` clears it, null is an explicit replacement, and
cross-kind replacement is allowed when the final application schema accepts it.
Under these ordinary rules, scalars, sequences, and null are atomic. An overlay
sequence replaces rather than extends a base sequence, `[]` clears it, null is an
explicit replacement, and cross-kind replacement is allowed when the final
application schema accepts it.
Replacing a subtree removes all displaced descendants from effective presence and
origin data; history retains the displaced definitions and identifies the parent
replacement. Surviving overlay paths have overlay origins, while recursively shared
mapping containers retain their contributing locations.
An overlay may not contain `configkit` anywhere, contribute the configured schema
version, or replace an ancestor of that version with a scalar or null. These rules
apply even if the supplied version text would be unchanged. Overlays cannot mask a
base parse, composition, or version failure because those checks run first.
An overlay may contain only root `configkit.sequences` declarations; overlay
imports and nested directives are rejected. Sequence declarations are removed
before application decoding. At declared or registered sequence destinations,
`replace`, `append`, and `upsert` use the same identity and policy rules as source
composition. Other overlay sequences remain atomic replacements. An overlay may
not contribute the configured schema version or replace an ancestor of that
version with a scalar or null. These rules apply even if the supplied version
text would be unchanged. Overlays cannot mask a base parse, composition, or
version failure because those checks run first.
## Explicit CLI overrides
@@ -875,10 +1101,16 @@ must never contain the raw argument or configured value. General environment
variable overlays are an explicit non-goal: environment access is reserved for
typed secret references.
Overrides apply after the selected overlay and before typed defaults/decoding, so
the input precedence is application defaults < composed base < selected overlay <
CLI. Recorded resolution, typed secret lookup, and semantic validation follow the
effective override tree. Overrides also work with `NoSource`.
Overrides are prepared after source assembly and inheritance, then assigned after
typed defaults and decoding and before application resolution. The order is:
defaults and decoded sources < CLI < application resolution. A CLI mapping
recursively overlays the decoded mapping; a CLI sequence replaces the complete
decoded sequence regardless of source collection operations. Inheritance and
typed defaulting do not run again after CLI assignment. Earlier parsing,
composition, inheritance, and decoding failures remain errors even if an override
would replace the failing value. Recorded resolution, typed secret lookup, and
semantic validation use the effective post-override values. Overrides also work
with `NoSource`.
Values are converted directly into ConfigKit's owned tree without YAML parsing or
string coercion. Supported values are `nil`, built-in strings and booleans, all
@@ -889,7 +1121,8 @@ false, and zero remain intentional; empty sequences replace whole lists; and map
values recursively overlay existing mappings. Map keys are sorted canonically and
sequence order is preserved.
All records are detached and validated before any is applied. Root and explicit
All records are detached and validated before any is applied. They cannot create
`configkit` or `extends` controls. Root and explicit
sequence-index paths are invalid; parsed numeric tokens remain mapping keys. A path
may create missing mapping ancestors but cannot traverse an existing scalar, null,
or sequence. Duplicate and ancestor/descendant-overlapping records are rejected.
@@ -906,8 +1139,12 @@ unsupported types, nonfinite floats, and limit excesses fail with safe `override
diagnostics. Diagnostics and provenance identify the safe `Flag` source, never raw
arguments or values.
Override origins are explicit CLI origins. History retains replaced base/overlay
definitions and records equal-value replacements as well as structural changes.
Override origins are explicit CLI origins. Resolution receives CLI values with
their explicit-presence and source information. Absence-based defaults preserve
those values, while recorded normalization and derivation may change them. History
retains replaced source definitions and records equal-value replacements as well
as structural changes. Secret lookup and validation use post-resolution values
with their effective source locations.
Caller-owned maps and slices are snapshotted during loading and must not be mutated
concurrently with the call. The executable `ExampleOverride` shows translating a
parser's explicit-set state into records while preserving an explicitly supplied
@@ -915,9 +1152,11 @@ zero.
## Explicit imports
Imports are opt-in. With `Options.Imports` enabled, a document may reserve its root
`configkit` mapping for an `imports` sequence. ConfigKit removes that mapping before
application decoding. Each sequence item is a mapping with exactly one `path`,
Imports are opt-in. With `Options.Imports` enabled, a document may use its root
`configkit` mapping for `imports` and `sequences` controls. Ordinary `Load` may use
`sequences` without enabling imports, but an `imports` field still requires
`Options.Imports`. ConfigKit removes recognized root controls before application
decoding. Each imports sequence item is a mapping with exactly one `path`,
`dir`, or `glob` selector. `path` and `dir` accept a string or a list of strings;
`glob` accepts one path pattern string. The optional `required` boolean controls
whether an empty selection or a missing named path is an error. Its default is
@@ -927,8 +1166,8 @@ strings. Every destination string is one literal mapping key and follows the
input-destination validation rules above; quote numeric-looking keys so YAML
reads them as strings. Null, scalar destinations, and non-string members are
`import.invalid`, as are all other unsupported shapes and fields. A `configkit`
key below the root is invalid. With imports disabled, any occurrence remains
`import.disabled`.
key below the root is invalid. With imports disabled, root imports and nested
`configkit` keys remain `import.disabled`.
The directive key is `configkit`, not the `_configkit` spelling used in v0.1.0.
There is no legacy alias: `_configkit` is ordinary application data, subject to
@@ -976,6 +1215,12 @@ filenames unrestricted; a non-nil empty slice selects none. Defaults do not make
a field applicable to selectors that do not support it, and load calls snapshot
the supplied pointers and slice before source acquisition.
A source's `configkit.sequences` declarations apply independently of import
enablement; a `configkit.imports` declaration still requires `Options.Imports`.
Sequence identities can be supplied either by these YAML declarations or by
application registrations. The [strict composition contract](#strict-document-composition)
defines their agreement, operation, and lifetime rules.
Acquisition visits declarations in authored order and each declaration's selected
paths in namespace-path lexical order, preserving source-record and version-callback
order. Ordinary mapping contributions in a main tier are assembled dependency-first:
@@ -987,9 +1232,10 @@ mapping scope, with helpers before the bodies that import them. Those local scop
do not use application collection policies and are not flattened into the outer
schedule. Item positions and diagnostic paths follow the resulting assembly order.
Each underlay group likewise acquires its inputs before dependency-ready assembly
in one strict scope. Independent groups remain separate precedence tiers; a higher
tier still replaces a lower tier's sequence values wholesale. Assembly order never
permits replacement.
in one strict scope. Independent groups remain separate precedence tiers; at a
declared or registered destination, the assembled higher sequence uses the
resolved collection operation against the lower sequence. Other sequences retain
whole-value replacement between tiers. Assembly order never permits replacement.
Acquisition order and resolution order serve different purposes: acquisition
assigns source records and callback order, while the shared scheduler determines
@@ -1020,13 +1266,17 @@ Each declaration may include `placement: mapping` or
including every match of a glob; placement is not inherited. Sequence-item
placement requires a nonempty effective destination and wraps the imported
mapping as one element of a sequence there. It does not split source sequences
or infer identity fields. Repeated sequences still conflict unless their path is
registered through `Definition.SequenceCompositions`; registration does not
change placement or precedence behavior. Sequence records retain source
locations through placement and history; indexes describe their assembled
positions while source coordinates continue to identify the declaring files.
Header/version policy and final application validation retain their existing
owners and timing; sequence registration is a strict-composition rule only.
or infer identity fields. Repeated sequences combine only when the destination
has an applicable YAML identity declaration or application registration; source
placement remains independent of collection identity and operation. Sequence
records retain source locations through placement and history; indexes describe
their assembled positions while source coordinates continue to identify the
declaring files.
Header/version policy retains its existing owner and timing. Application
registrations also enforce identity on the final effective typed configuration;
YAML-only declarations are enforced at structural combination and inheritance
boundaries. The [configuration contract](#strict-document-composition) defines
these validation boundaries.
An item import first opens a local mapping scope. Its body and ordinary helper
imports compose within that record. An omitted helper destination or

View File

@@ -21,7 +21,7 @@ sanitized shapes using only ConfigKit and the Go standard library:
| Named presets | One-load sibling inheritance, explicit false, nested destination resolution, and source/derivation inspection | Choosing and executing a preset |
| Effective configuration diff | Same-type frozen effective comparisons for two selected files or base-versus-overlay impact | Source/options selection and consumer-owned safe presentation |
| Shared source-version checks | One `Definition.Version` policy for accepted declarations across participating source documents, with optional or required declarations and sanitized evidence | Choosing which declarations the application accepts; final assembled schema and domain validation |
| Identified prompt sequences | Explicit item path/glob imports, helper-supplied IDs, reusable single-field registration, and main-tier replacement of a lower-priority collection | Prompt meaning and final domain validation |
| Sequence catalogs | Declared collection operations, record inheritance, typed CLI and resolution, inspection, exact and wildcard application registrations | Catalog meaning and final domain validation |
The YAML under [`testdata`](../../examples/consumers/testdata) is loaded by the
executable examples and ordinary tests. It contains no private application data,
@@ -55,6 +55,21 @@ both without redefining their syntax. ConfigKit overlays are configuration overl
they do not select or replace an application's LLM, model, pipeline, report, or
execution profile concepts.
## Explicit main-file discovery
Applications choose the search conventions and their precedence, then pass the
resulting `SearchConfig` source to the same `Load` or `LoadLayered` entry point.
For example, the maintained sequence catalog puts a caller-selected file before
a custom directory fallback while keeping its underlay and overlay choices
independent. See [`sequence_catalog.go`](../../examples/consumers/sequence_catalog.go)
and its [behavior tests](../../examples/consumers/sequence_catalog_test.go), plus
the [working-directory executable example](../../examples/consumers/discovery_test.go).
Together, the root-owned [discovery API](../../discovery.go) and [configuration
discovery contract](../config.md#configuration-discovery) define ordered
concatenation, directory-first Cartesian expansion, optional exhaustion, and
selected-file behavior. [ADR 0019](../adr/0019-explicit-configuration-discovery.md)
records why discovery is explicitly selected.
## Import defaults and selection inspection
`Options.ImportDefaults` and `LayeredOptions.ImportDefaults` apply one
@@ -184,36 +199,35 @@ import, source-location, and ownership behavior.
## Identified prompt sequences
[`LoadPromptCatalog`](../../examples/consumers/prompt_catalog.go) uses one
definition registration for the literal `prompts` path and `id` field. Its
load-wide filename defaults accept two patterns, an explicit directory
declaration overrides those patterns, and an exclusion omits one matching
candidate. A nested ordinary helper import supplies an item's ID. A separate
empty optional directory search is read from `SourceInfo.ImportSelections`.
The executable example prints both the scheduled prompt order and the single
empty-search outcome; the behavior test also follows a mounted resource through
its effective metadata origin to the declaring source context. The completed
main sequence replaces the lower-priority default collection as one value.
The maintained [executable example and behavior tests](../../examples/consumers/prompt_catalog_test.go)
show the scheduled result, empty-search inspection, excluded candidate, and
resource lookup without a consumer-side discovery pass. The corresponding
[complex fixture](../../examples/consumers/testdata/prompt-catalog/config.yml)
and [simple required-file fixture](../../examples/consumers/testdata/prompt-catalog/simple.yml)
are complete local examples.
[`LoadPromptCatalog`](../../examples/consumers/prompt_catalog.go) shows the
smaller import-focused adoption path: a registered `prompts.id` identity,
directory and glob selection, a helper import that supplies an item's ID, and
resource lookup through the effective source context. Its executable examples
and tests are in
[`prompt_catalog_test.go`](../../examples/consumers/prompt_catalog_test.go), with
complete [catalog](../../examples/consumers/testdata/prompt-catalog/config.yml)
and [required-file](../../examples/consumers/testdata/prompt-catalog/simple.yml)
fixtures.
[`LoadRequiredPromptCatalog`](../../examples/consumers/prompt_catalog.go) and
[`Example_requiredPromptCatalog`](../../examples/consumers/prompt_catalog_test.go)
keep the smaller adoption path visible: one explicitly required file import,
the same identity registration, and the same application validation.
[`LoadSequenceCatalog`](../../examples/consumers/sequence_catalog.go) integrates
an underlay declaration, imported main fragments, identified-record inheritance,
selected overlay `append`, `upsert`, and `replace`, a typed CLI override, and
recorded application resolution. Its tests and runnable example demonstrate
stable output order, whole-record upsert, provenance/history inspection,
resource lookup for inherited records, and secret-safe explanation in
[`sequence_catalog_test.go`](../../examples/consumers/sequence_catalog_test.go).
[`LoadNamespacedMapCatalog`](../../examples/consumers/sequence_catalog.go) adds
an exact root registration and a wildcard namespace registration, showing that
each concrete sequence has an independent identity scope. The same tests show
that a YAML identity declaration without an application registration does not
create final-result identity enforcement.
The example's `validatePromptCatalog` remains application-owned. It checks that
prompt names and message lists satisfy this consumer's requirements after
ConfigKit has composed, projected, and decoded the configuration. It does not
add recursive record merging, identity-based overlay, or final uniqueness checks
after defaults and resolution. See the [strict composition contract](../config.md#strict-document-composition),
[import syntax](../config.md#explicit-imports), and
[ADR 0014](../adr/0014-sequence-placement-and-composition.md) for the exact
boundaries.
The examples keep prompt meaning and domain checks in the consuming application.
The [configuration contract](../config.md#strict-document-composition) owns
declaration, identity, operation, ownership, and final-validation semantics;
[sibling inheritance](../config.md#sibling-inheritance) defines record
relationships, and the [CLI contract](../config.md#explicit-cli-overrides) owns
override ordering.
Known module catalogs should use canonical object entries with a module identifier
and typed options. A small union struct is sufficient when the available option

View File

@@ -63,6 +63,7 @@ Accepted decisions:
- [Separate import selection controls](adr/0016-independent-import-controls.md)
- [Define identity and merge rules for each collection](adr/0017-collection-sequence-policy.md)
- [Apply CLI overrides after inheritance and decoding](adr/0018-cli-after-structural-resolution.md)
- [Keep main-file discovery explicit and composable](adr/0019-explicit-configuration-discovery.md)
For cross-cutting changes, follow every applicable row. Completed implementation
and remediation plans are preserved in Git history at `cb5b435`; use current
@@ -135,12 +136,13 @@ Current behavior belongs to the configuration contract's
[input destinations](config.md#input-destinations),
[imports](config.md#explicit-imports), and
[shared source-version checks](config.md#shared-source-version-checks) sections.
Explicit sequence-item placement and registered strict composition are owned by
the [strict composition](config.md#strict-document-composition) and
[imports](config.md#explicit-imports) sections. The
[maintained prompt catalog](consumers/README.md#identified-prompt-sequences)
shows the current consumer boundary, and [ADR 0014](adr/0014-sequence-placement-and-composition.md)
records the decision.
Explicit sequence-item placement, collection operations, identity registration,
and final registered-identity validation are owned by the
[strict composition](config.md#strict-document-composition) section;
[imports](config.md#explicit-imports) owns acquisition and scheduling. The
[maintained sequence catalog](consumers/README.md#identified-prompt-sequences)
shows the consumer boundary, and [ADR 0017](adr/0017-collection-sequence-policy.md)
records the collection decision.
The completed sequence-composition roadmap, implementation plan, and resolved
remediation record are preserved in Git history at `008d2ba` under
@@ -166,6 +168,27 @@ The subsequent import-integration audit and completed remediation record are
preserved at `6903c9e:docs/roadmap/remediation.md`; retrieve them with
`git show 6903c9e:docs/roadmap/remediation.md`.
The completed sequence-identity/collection-policy roadmap and implementation
plan are preserved in Git history at `d0a605a`:
```sh
git show d0a605a:docs/roadmap/sequences.md
git show d0a605a:docs/roadmap/implementation.md
```
Current behavior belongs to the configuration contract's
[strict composition](config.md#strict-document-composition),
[sibling inheritance](config.md#sibling-inheritance), and
[CLI overrides](config.md#explicit-cli-overrides) sections.
The completed remediation record, including the algorithm-audit findings, is
preserved at `ea46001:docs/roadmap/remediation.md`.
The completed configuration-discovery roadmap, implementation plan, and
remediation record are preserved in Git history at `0369111`, for example
`git show 0369111:docs/roadmap/remediation.md`. Current behavior belongs to
[configuration discovery](config.md#configuration-discovery); the public
[discovery declarations](../discovery.go) own the Go API.
Potential enhancements are tracked in [future features](roadmap/future.md).
Install the Go version required by `go.mod` or a compatible newer toolchain.
@@ -215,9 +238,11 @@ code changes. Use the component overview to find the corresponding implementatio
| --- | --- | --- |
| YAML syntax, diagnostics, parser limits | [Adapter tests](../internal/yaml/parse_test.go), [source translation](../internal/engine/parse_test.go) | `go test ./internal/yaml` and `go test ./internal/engine -run '^TestParseSource'` |
| File/reader access, imports, globs, containment | [Sources](../internal/engine/source_test.go), [FIFO](../internal/engine/source_fifo_test.go), [imports](../internal/engine/imports_test.go), [relative destinations](../internal/engine/destination_import_test.go), [globs](../internal/engine/glob_test.go) | `go test ./internal/engine -run 'Source\|Import\|Glob\|Filesystem\|Nonregular\|RegularFile'` |
| Explicit main-file discovery and absent-input loading | [Candidate selection](../internal/engine/discovery_test.go), [load integration](../internal/engine/discovery_load_test.go), [public facade](../facade_discovery_test.go), and maintained [catalog](../examples/consumers/sequence_catalog_test.go) and [preset](../examples/consumers/discovery_test.go) examples | `go test ./internal/engine -run 'Discovery|Search'`, `go test . -run 'Search'`, and `go test ./examples/consumers -run 'WorkingDirectoryPreset|SequenceCatalog'` |
| Import defaults, empty-selection evidence, and dependency-ready ordering | [Defaults and facade evidence](../facade_import_defaults_test.go), [engine import evidence](../internal/engine/import_evidence_test.go), [scheduler](../internal/engine/dependency_order_test.go), [scheduling integrations](../internal/engine/imports_test.go) | `go test ./internal/engine -run 'Import|ScheduleDependencies|Inheritance'`, `go test . -run 'ImportDefaults|FacadeImport'`, and `go test ./examples/consumers` |
| Native filesystem and mapping sources | [Native acquisition](../internal/engine/native_filesystem_test.go), [filesystem facade](../facade_filesystem_test.go), [mapping facade](../facade_mapping_test.go) | `go test ./internal/engine -run 'NativeFilesystem'` and `go test . -run 'TestFacade'` |
| Composition, selected overlays, CLI precedence | [Composition](../internal/engine/compose_test.go), [overlay](../internal/engine/overlay_test.go), [selected overlays](../internal/engine/selected_overlay_test.go), [overrides](../internal/engine/override_test.go) | `go test ./internal/engine -run 'Compose\|Overlay\|Override'` |
| Sequence identity, declarations, and operations | [Identity](../internal/engine/sequence_identity_test.go), [registrations](../internal/engine/sequence_registration_test.go), [declarations](../internal/engine/sequence_declaration_test.go), [combination](../internal/engine/sequence_combine_test.go), [final facade validation](../facade_final_sequence_identity_test.go), and [consumer catalog](../examples/consumers/sequence_catalog_test.go) | `go test ./internal/engine -run 'Sequence\|Collection'`, `go test . -run 'SequenceIdentity\|SequenceRegistration'`, and `go test ./examples/consumers` |
| Sibling inheritance, provenance, history, sensitivity | [Expansion](../internal/engine/inheritance_test.go), [destination scopes](../internal/engine/inheritance_walk_test.go), [origins](../internal/engine/inheritance_origin_test.go), [history](../internal/engine/inheritance_history_test.go), [secrets](../internal/engine/inheritance_secret_test.go), [public loading](../facade_inheritance_test.go) | `go test ./internal/engine -run 'Inheritance\|Inherited'` and `go test . -run '^TestFacadeInheritance\|^TestFacadeInheritedSecrets'` |
| Layered loading and source equivalence | [Grouped lifecycle](../internal/engine/load_request_test.go), [layered facade](../facade_layered_test.go), [equivalence](../facade_equivalence_test.go) | `go test ./internal/engine -run '^TestGrouped'`, `go test . -run '^TestFacadeLayered'`, and `go test . -run '^TestEquivalentSourceForms'` |
| Input and relative-import destinations, mounted sources, and source ownership | [Destination engine tests](../internal/engine/destination_test.go), [tier tests](../internal/engine/destination_tiers_test.go), [import traversal and version tests](../internal/engine/destination_import_test.go), [accounting tests](../internal/engine/destination_accounting_test.go), [lifecycle inspection tests](../internal/engine/destination_import_lifecycle_test.go), [public source/error tests](../facade_destination_test.go), [resource contexts](../facade_resource_context_test.go), and [consumer examples](../examples/consumers/destinations_test.go) | `go test ./internal/engine -run 'Destination|Import|Glob|Mounted'`, `go test . -run 'Destination|Import|SourceContext|FacadeMounted'`, and `go test ./examples/consumers` |
@@ -230,6 +255,7 @@ code changes. Use the component overview to find the corresponding implementatio
| Effective configuration diff | [Engine comparison and masking tests](../internal/engine/diff_test.go), [engine operation tests](../internal/engine/diff_operation_test.go), [root facade tests](../facade_diff_test.go), and [consumer examples](../examples/consumers/diff_test.go) | `go test ./internal/engine -run 'Diff|Explain|History|Secret|Independent'`, `go test . -run 'Diff|FacadeInspection|PublicFacadeBoundary|DependencyImportBoundary'`, `go test ./examples/consumers`, and `go test . -run '^ExampleDiff$'` |
| Consumer examples and migration fixtures | [Consumer tests](../examples/consumers/consumers_test.go), [native, layered, and ordinary relative-import examples](../examples/consumers/layered_test.go), [mounted collection examples](../examples/consumers/destinations_test.go), [API examples](../example_test.go) | `go test ./examples/consumers` and `go test . -run '^Example'` |
| Assembly and layered scaling evidence | [Assembly benchmarks](../internal/engine/assembly_bench_test.go), [layered benchmarks](../internal/engine/underlay_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkUnderlay' -benchmem`; repeat with `^BenchmarkCompose` and `^BenchmarkOverlay` |
| Identified-sequence scaling evidence | [Mixed-upsert benchmark](../internal/engine/sequence_combine_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkSequenceMixedUpsert$' -benchmem` |
| Inheritance scaling evidence | [Expansion benchmarks](../internal/engine/inheritance_bench_test.go) | `go test ./internal/engine -run '^$' -bench '^BenchmarkInheritance' -benchmem` |
| Documentation links and dependency isolation | [Link checks](../documentation_test.go), [import boundary](../import_boundary_test.go), [public boundary](../public_boundary_test.go) | `go test . -run 'TestLocalDocumentationLinks\|TestDependencyImportBoundary\|TestPublicFacadeBoundary'` |
| Development check behavior | [Development checker test](../development_check_test.go) | `go test . -run '^TestDevelopmentCheckHandlesFilenamesAndStopsOnFailure$'` |

View File

@@ -65,11 +65,18 @@ returns a closed cycle rotated to its smallest key; blocked dependents are not
reported as cycle members.
Import composition schedules a main tier or one underlay group as a single
strict-scope graph. Item contributions own nested local strict-mapping scopes;
their helpers complete before the item is placed in its outer scope. Inheritance
uses the same ready-node rule inside each sibling mapping scope, while preserving
the authored mapping order in its output. These scopes do not flatten into one
global graph, and independent precedence tiers remain separate.
strict-scope graph. Applicable collection declarations are gathered for the
scope before peer composition, including identity inherited from lower tiers.
Identified peer sequences then combine distinct records in assembled source and
list order; after the strict scope completes, layered assembly applies the
resolved collection operation against the lower value, as defined by the
[configuration contract](../config.md#strict-document-composition). Item
contributions own nested local strict-mapping scopes; their helpers complete
before the item is placed in its outer scope. Inheritance uses the same
ready-node rule inside each sibling mapping scope, while preserving the authored
mapping order in its output.
These scopes do not flatten into one global graph, and independent precedence
tiers remain separate.
The main owners are [`import_selection.go`](../../internal/engine/import_selection.go),
[`directory_walk.go`](../../internal/engine/directory_walk.go),

View File

@@ -7,14 +7,18 @@ private state; public signatures and fields do not expose engine types.
Implemented component owners:
- `internal/engine` owns host and native-filesystem source acquisition,
grouped strict composition, recursive precedence assembly, per-input mapping
and sequence-item placement, definition-registered distinct-ID sequence
composition within strict tiers, and immediate-parent-relative import
placement within that assembly. Higher-priority sequences replace lower
sequences as whole values. The engine also owns dependency-ready,
destination-local sibling inheritance before typed decoding, plus shared
source-version checking with sanitized declaration evidence.
- `internal/engine` owns host and native-filesystem source acquisition and
explicitly selected configuration discovery. A search resolves once during
shared acquisition; the selected concrete file or absent-input state then
follows the existing composition, metadata, and resource-context lifecycle.
It also owns grouped strict composition, recursive precedence assembly,
per-input mapping and sequence-item placement, YAML collection declarations
and operations for identified sequences, and application identity registrations for strict
composition, inheritance, and final validation. It applies the selected
collection operation at precedence boundaries and owns dependency-ready,
destination-local inheritance before typed decoding. CLI assignment follows
inheritance and decoding, before application resolution. The engine also owns
shared source-version checking with sanitized declaration evidence.
Explicit import discovery supports declared paths, directory searches, and
globs; load-wide defaults and source-local empty-selection evidence cross the
root facade without changing acquisition order. Import and inheritance
@@ -30,9 +34,9 @@ Implemented component owners:
or records. Callback wrappers pass only live engine context during a callback;
`Result.Config` is detached and its private inspection remains frozen
independently.
- Root conversion files explicitly cross the path, record, callback, secret, and
error boundaries. There are no public aliases, anonymous engine embeddings, or
public engine bridge methods.
- Root conversion files explicitly cross the path, record, callback, secret,
search-description, and error boundaries. There are no public aliases,
anonymous engine embeddings, or public engine bridge methods.
- Root `Diff` and `Difference` provide the public comparison boundary and
convert detached engine records to root-owned paths and snapshots; consumer
examples own presentation of the safe changed regions.
@@ -52,13 +56,16 @@ these constraints and rejects exported aliases, fields, or signatures involving
internal types.
The [configuration contract](../config.md) owns implemented YAML and lifecycle
semantics, including [input destinations](../config.md#input-destinations),
semantics, including
[configuration discovery](../config.md#configuration-discovery),
[input destinations](../config.md#input-destinations),
[shared source-version checks](../config.md#shared-source-version-checks), and
[effective configuration diffs](../config.md#effective-configuration-diffs).
The [strict composition contract](../config.md#strict-document-composition)
owns sequence placement, registration, replacement, and history behavior; the
[consumer guide](../consumers/README.md#identified-prompt-sequences) shows the
maintained example.
owns sequence declarations, identity, operations, ownership, and final
validation; [CLI overrides](../config.md#explicit-cli-overrides) own override
ordering. The [consumer guide](../consumers/README.md#identified-prompt-sequences)
links maintained workflow examples.
The [consumer adoption guide](../consumers/README.md) owns application guidance.
The [architecture policy](../policy/architecture.md) owns contributor
boundaries; [ADR 0006](../adr/0006-public-facade.md) records the facade decision
@@ -74,5 +81,9 @@ records the source-version decision.
sequence placement and composition. [ADR 0015](../adr/0015-shared-dependency-ready-ordering.md)
records the shared dependency-ready ordering decision; [ADR
0016](../adr/0016-independent-import-controls.md) records the independent
selection controls. The [import internals](imports.md) document candidate and
namespace identities, discovery and scheduling, and the bounded algorithms.
selection controls. [ADR 0017](../adr/0017-collection-sequence-policy.md) and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md) record collection
operations and CLI ordering. [ADR 0019](../adr/0019-explicit-configuration-discovery.md)
records explicit main-file discovery. The [import internals](imports.md)
document candidate and namespace identities, discovery and scheduling, and the
bounded algorithms.

View File

@@ -32,11 +32,11 @@ structure as implemented when its changes land.
Constraints must reflect deliberately selected requirements and decisions.
Do not promote incidental implementation choices into architectural invariants.
The collection-policy and CLI-ordering requirements below reflect accepted
[ADR 0017](../adr/0017-collection-sequence-policy.md) and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md). The
[sequence roadmap](../roadmap/sequences.md) tracks the work to implement them;
the configuration contract continues to describe implemented behavior.
The implemented collection-policy and CLI-ordering constraints below reflect
accepted [ADR 0017](../adr/0017-collection-sequence-policy.md) and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md). The configuration
contract owns their current behavior; these architectural constraints describe
the system boundaries and guarantees that guide changes to that behavior.
## Success Criteria
@@ -144,8 +144,12 @@ remains in the roadmap until implemented.
ConfigKit owns configuration input processing and the coordination of decoding,
resolution, validation, and inspection. Applications own their configuration
types, field meanings, defaults, normalization, semantic validation, and schema
version policy. Applications also own main-file search conventions, CLI wiring,
remote acquisition, runtime initialization, and resource checks.
version policy. Applications choose and order their main-file search conventions;
ConfigKit provides explicitly selected search descriptions that resolve during
loading. Applications also own CLI wiring, remote acquisition, runtime
initialization, and resource checks. See the [configuration discovery
contract](../config.md#configuration-discovery) and [ADR
0019](../adr/0019-explicit-configuration-discovery.md).
The engine applies each input's mapping destination within the existing source
assembly, applies immediate-parent-relative YAML destinations to imports, and
@@ -157,11 +161,13 @@ placement, and [ADR 0012](../adr/0012-relative-import-destinations.md) records
the import extension.
Library code must not depend on consuming applications or initialize their
runtime resources. It must not perform implicit network access, discover
application-specific paths, or change process-global state during ordinary `Load`.
runtime resources. It must not perform implicit network access, implicitly
discover application-specific paths, or change process-global state during
ordinary `Load`.
Import discovery is available only through declarations in selected sources:
explicit `path`, `dir`, and `glob` selectors may search their declared scope, but
ConfigKit does not search conventional application paths or discover a main
ConfigKit does not search conventional application paths unless the application
explicitly supplies a discovery description, and it never discovers a main
configuration file implicitly. Imports and sibling inheritance share
dependency-ready scheduling within their existing scopes; acquisition order and
precedence-tier boundaries remain distinct. The [configuration contract](../config.md#explicit-imports)
@@ -373,8 +379,8 @@ as records survive, move, or are replaced. Identity diagnostics use paths and
available origin information without exposing identity values. The guarantee
requires no new mutation-history system or exact reconstruction of causes for
records supplied by defaults or callbacks. Exact identity equality, declaration
syntax, presence cases, and operation tables belong in the configuration contract
when implemented; the roadmap owns those details until then.
syntax, presence cases, and operation tables belong in the
[configuration contract](../config.md#strict-document-composition).
## Ownership And Confidentiality

77
docs/releases/v0.10.0.md Normal file
View File

@@ -0,0 +1,77 @@
# ConfigKit v0.10.0
This minor release adds explicit, composable discovery of a configuration file.
Applications choose the search conventions and their order; ConfigKit selects one
file during loading and returns the final configuration with its normal metadata.
Optional searches that find no file share the expanded `NoSource()` lifecycle.
## Changes
- Add `SearchConfig`, immutable `Search` and `FilenameGroup` descriptions, and
search-wide requiredness through `SearchOptions`. Compose explicit `Paths`,
custom `SearchDirectory` descriptions, and built-in Unix system, user-config,
home, or working-directory presets with `Searches`. See the
[public declarations](../../discovery.go) and
[discovery contract](../config.md#configuration-discovery).
- Expand searches in supplied order, trying a directory's filename groups before
moving to the next directory. Named and generic groups provide conventional
YAML filenames; custom groups provide literal names. Directory searches
distinguish omitted filename selection from an explicitly empty selection.
Constructors perform no filesystem access, and each load observes current
directory bases and file availability.
- Select the first existing candidate. Access errors and failures while loading
the selected file terminate the load rather than trying another configuration.
Selected files retain physical source identity, placement, provenance, imports,
and relative-resource context. Discovery works with ordinary loading and
layered inputs, including underlays, without adding a precedence tier.
- Permit enabled imports with an absent main input: there are no main import
declarations to traverse, while actual underlay sources retain normal imports.
Support ordinary selected overlays with an absent main and explicit filesystem
context, including a root-only host context. Layered overlays retain their
independent `OverlayFilesystem` requirement. See
[single-source loading](../config.md#single-source-loading) and
[layered loading](../config.md#layered-loading).
- Preserve ordinary file-context behavior after discovery and include the
failing candidate's file identity in access diagnostics. Use Go's
platform-aware Windows component rules and native drive-root-relative path
resolution. Isolate user-config tests according to the host Go conventions.
## Compatibility and upgrade
Existing explicit source constructors and loading APIs remain available.
`File(path)` remains an exclusive selection that fails for a missing file;
discovery is activated only by explicitly supplying `SearchConfig`.
Searches are optional by default. Set `SearchOptions{Required: true}` when the
application must find a file. An exhausted optional search contributes no main
document or source record; application version checks, defaults, resolution,
and validation still run under their existing rules and may reject incomplete
configuration. A selected overlay is still required to exist.
The former `NoSource()` rejections of enabled imports and explicit filesystem
context are relaxed. An active ordinary overlay with no main file now loads
when explicit context is supplied. Applications that relied on those combinations
being rejected should review their option handling. Selected host files derive
their own base directory; do not supply an explicit `BaseDir` or native filesystem
context for a search that selects a host file.
Applications may replace their own fallback-loading loops with the new search
descriptions. The maintained [consumer examples](../consumers/README.md#explicit-main-file-discovery)
show simple preset selection and custom-first discovery alongside existing
configuration tiers. Composition, collection operations, inheritance, CLI
precedence, and the complete-on-return ownership model retain their existing
semantics.
Select this version and rebuild:
```sh
go get gitea.maximumdirect.net/eric/ConfigKit@v0.10.0
```
The consumer minimum remains Go 1.26.0. Release validation uses Go 1.26.10.
The YAML dependency remains `go.yaml.in/yaml/v4 v4.0.0-rc.6`, and the maintainer
tool remains ReleaseKit v0.4.1. Linux amd64 and arm64 remain release-validated
targets; macOS remains best-effort. Publication follows the existing
[release procedure](../release.md).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.10.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.10.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.10.0/docs/oss/attribution.md)

111
docs/releases/v0.9.0.md Normal file
View File

@@ -0,0 +1,111 @@
# ConfigKit v0.9.0
This minor release expands import selection, adds author-declared collection
identity and operations, and supports inheritance between identified records.
It also moves CLI assignments after inheritance and typed decoding. These changes
include breaking API and configuration behavior changes from v0.8.0; review the
migration guidance below before upgrading.
## Changes
- Extend imports with named-file and directory lists, directory filename patterns,
exclusions, recursion and depth controls, and explicit hidden-file and symlink
discovery controls. Add `ImportDefaults` to ordinary and layered options, with
authored declarations taking precedence over application defaults. Imports are
optional by default. `SourceInfo.ImportSelections` exposes completed optional
declarations that selected no sources. See the
[import contract](../config.md#explicit-imports) and
[consumer guidance](../consumers/README.md#import-defaults-and-selection-inspection).
- Use shared dependency-ready ordering for import composition and sibling
inheritance. Dependencies complete before dependents, with deterministic
selection among currently eligible contributions. Source acquisition and
callback order remain separate from assembly order. See the
[ordering decision](../adr/0015-shared-dependency-ready-ordering.md).
- Add root `configkit.sequences` declarations for collection identity and
`replace`, `append`, or `upsert` operations, including in selected overlays.
Strict peers combine distinct identities and reject duplicates; precedence
and inheritance apply the destination's collection operation. Upsert replaces
a whole matching record at its retained position. Identity supports compound
field sets and scalar strings, numbers, booleans, and null. See the
[collection contract](../config.md#strict-document-composition).
- Add `Definition.SequenceIdentities`, exact `SequenceIdentityAt` registrations,
and dynamic `SequenceIdentityMatching` selectors. Application registrations
govern structural collection operations and check existing matching final
typed sequences, including values from defaults, CLI assignments, and
application resolution. YAML-only declarations govern structural operations
without adding that final-result check.
- Support record inheritance within an identified sequence using an `extends`
mapping containing the parent's complete identity. The child retains its own
identity, relationships use surviving assembled records, and output sequence
order is preserved. See [sibling inheritance](../config.md#sibling-inheritance)
and the maintained [sequence catalog](../consumers/README.md#identified-prompt-sequences).
- Apply explicit CLI overrides to the decoded configuration before application
resolution. Mappings retain omitted fields and their presence metadata;
sequences replace whole lists independently of collection operations. See the
[CLI contract](../config.md#explicit-cli-overrides).
- Correct native import alias and contained-symlink handling, source-version
path projection, and provenance/history through collection movement,
replacement, inheritance, and CLI assignment. Reduce repeated scans,
accumulated policy copying, and sparse-projection storage during construction.
## Compatibility and upgrade
This pre-v1 minor release intentionally changes the v0.8.0 contracts:
1. **Migrate import requiredness.** The YAML `optional` field is removed and is
rejected. Replace `optional: true` with `required: false`; omission is equivalent
when the application does not default imports to required. Replace
`optional: false` with `required: true`. Imports that previously relied on
implicit requiredness now need `required: true`, or an application
`ImportDefaults.Required` default set to true. Review discovery filters:
`follow` and `hidden` default to true, and directory searches select all
filenames unless a pattern is supplied.
2. **Replace sequence registrations.** `SequenceComposition` and
`Definition.SequenceCompositions` are removed without compatibility aliases.
For a previous registration at `prompts` using `id`, use:
```go
rules.SequenceIdentities = []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
}
```
These registrations now also enforce identity in final typed sequences. Fix
missing or duplicate identities introduced by defaults, CLI assignments, or
application resolution. Registration does not require a collection to exist
or determine its allowed shape; schema and application validation retain
those responsibilities.
3. **Review collection precedence and ordering.** An identified collection now
defaults to upsert across layers and inheritance when no explicit mode
applies, rather than unconditional whole-sequence replacement. Declare
`mode: replace` at the relevant destination where replacement is intended.
With append or upsert, `[]` contributes no records and does not clear the
collection; null is invalid. Use `mode: replace` with `[]` to clear it.
An explicit mode persists at its destination until changed; a parent's mode
does not transfer to a different inheriting destination. Unidentified
sequences retain strict peer conflicts and whole-value precedence
replacement. Dependency-ready assembly can change sequence positions and
diagnostic or history order compared with the former traversal order.
4. **Review CLI recovery and inheritance assumptions.** CLI overrides can no
longer repair values that fail earlier decoding, bypass source or inheritance
errors, or add or cancel inheritance relationships. Supply structural
directives in source configuration. Application resolution still receives
explicit CLI values and may record subsequent normalization or derivation.
5. **Update affected Go literals and diagnostic handling.** Use keyed literals
for expanded option and source-information structs. Collection validation adds
`CodeSequenceInvalid` (`sequence.invalid`) and `CodeSequenceConflict`
(`sequence.conflict`); update diagnostic handling where applicable.
Select this version and rebuild:
```sh
go get gitea.maximumdirect.net/eric/ConfigKit@v0.9.0
```
The consumer minimum remains Go 1.26.0. The YAML dependency remains
`go.yaml.in/yaml/v4 v4.0.0-rc.6`, and the maintainer tool dependency remains
ReleaseKit v0.4.1. Linux amd64 and arm64 remain release-validated targets;
macOS remains best-effort. Publication follows the existing
[release procedure](../release.md).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.9.0/docs/oss/attribution.md)

222
docs/roadmap/complexity.md Normal file
View File

@@ -0,0 +1,222 @@
# Internal Complexity Refactoring Roadmap
Status: proposed target state. This roadmap defines a behavior-preserving
refactor; a later implementation plan will define stages and exact private
signatures and file names.
## Purpose
Make ConfigKit's import, inheritance, and projection implementation easier to
understand, review, and change while preserving its public APIs and configuration
behavior. Concentrate on functions that coordinate several responsibilities and
carry substantial shared state.
The target is clearer responsibility and ownership. Adding a few well-named
helpers, private records, or focused files can serve that target even when total
line count increases. Complexity measurements help locate candidates; they do
not establish function-size limits, score thresholds, or release gates.
The [architecture policy](../policy/architecture.md) owns package and ownership
constraints. The [configuration contract](../config.md) and exported Go comments
remain authoritative for consumer behavior.
## Current Baseline
Reviewed baseline: `7a83704f43ef9870db33661231cdff5d6ee502f7`.
The root facade delegates to a cohesive private engine, supported by the ordered
document representation and YAML adapter. Most functions are small. The
complexity audit identified these concentrated maintenance costs:
| Area | Current implementation | Required improvement |
| --- | --- | --- |
| Imports | `imports.go` spans acquisition, item-local composition, and evidence projection. Private loading variants pass long positional argument lists. Source admission and directory selection contain substantial mixed control flow. | Give import entry points explicit, cohesive inputs and separate the responsibilities of acquisition, scope assembly, and evidence projection. |
| Inheritance | `applyInheritanceScopeTemplates` coordinates payload preparation, parent combination, policy ownership, origins, history, and accounting in one executor. | Keep one readable executor with focused operations and explicit state handoffs. |
| Projection | `valueOperation.project` combines type dispatch with struct flattening, map and sequence traversal, scalar conversion, sensitivity, and accounting. | Retain clear dispatch while moving substantial value-kind handling into focused helpers. |
The implemented inventory belongs to the
[component overview](../internal/overview.md). Existing import flow and scheduling
are described in [import internals](../internal/imports.md).
## Target Architecture
The refactor stays within `internal/engine`. The root facade, `internal/document`,
and `internal/yaml` retain their existing responsibilities and dependency
direction. Focused files and ordinary private functions express the new internal
organization. Package extraction is outside this roadmap's selected design.
`executeLoadRequest` remains the visible coordinator of the shared load lifecycle.
Its phase order should remain easy to follow. Helpers expose meaningful work and
results rather than distributing that order across a generic execution framework.
Existing shared mechanisms remain shared: recursive mapping combination,
identified-sequence operations, dependency-ready scheduling, schema descriptors,
and projection/accounting state. New helper boundaries make their ownership and
use clearer without creating parallel implementations.
### Imports: Cohesive Inputs And Scope Ownership
Import loading has a clear private entry surface using named request/context
fields for related inputs. Reuse the existing load-request, acquisition-scope,
and traversal models where they fit. Consolidate forwarding variants whose only
purpose is passing another optional positional argument through the chain.
Retain convenience helpers when they express a useful, distinct operation.
Keep these state lifetimes identifiable:
- Load-wide state: accounting, shared inclusion/repetition information,
namespaces, and source-version processing.
- Acquisition requests: source and filesystem context, placement, declarations,
applicable policy, and the relevant assembly handoff.
- Composition scopes: peer contributions, item-local helpers, record ownership,
and pending declarations.
- Evidence: physical source records, effective destinations, origins, and retained
history projections.
Group acquisition, pending-item assembly, and evidence projection in focused
files. A caller should be able to identify which operation acquires resources,
which composes already acquired content, and which projects retained evidence.
Shared mutable state has a visible owner; a request record need not duplicate
state already held by that owner.
Within dense admission, directory-selection, and declaration-parsing routines,
extract coherent operations where that reduces nested, mixed responsibilities.
Preserve the existing selection algorithms and host/native filesystem behavior.
Resource ownership and closure remain explicit across success and failure paths.
The resulting flow preserves the distinction among acquisition order, strict
scope scheduling, and precedence. Completing a helper scope before placing its
item remains part of the same import flow. Ordinary and layered loading continue
to share the applicable mechanisms.
### Inheritance: One Executor With Explicit Phases
Keep the mapping and identified-sequence adapters and their shared template
executor. The executor coordinates focused operations for:
- Preparing child payload, directives, and destination-local state.
- Preparing the resolved parent payload and applicable inherited policy.
- Combining values through the existing mapping and sequence mechanisms.
- Producing policy, origin, replacement, and replay evidence from that combination.
- Accounting for expansion and retained history under the existing budgets.
These are responsibility boundaries, not a required helper count or a new
scheduler. Small private result records are appropriate when related outputs
otherwise travel through long argument lists or implicit mutations.
Authored templates, resolved values, scoped policy, and inspection evidence remain
distinguishable. Immutable borrowed inputs and owned mutable results have clear
lifetimes. Nested instructions remain available for destination-local evaluation.
For identified-record inheritance, the child's own identity fields and origins
remain intact; the parent's identity fields are excluded from inherited payload
and its history, as the current contract requires.
One sequence-operation result continues to drive value construction, record
ownership, origins, and historical position projection. Helpers consume that
evidence directly. Scheduling remains dependency-ready, enclosing relationships
resolve before nested ones, and assembled sequence order remains unchanged.
### Projection: Explicit Value-Kind Handlers
Keep `projectConfiguration` as the schema-checked entry and `valueOperation` as
the owner of per-operation traversal and accounting state. The central dispatch
clearly selects the appropriate handling for pointers, dynamic values, structs,
maps, sequences, and scalars.
Substantial struct, map, and sequence work lives in focused handlers. Struct
projection retains declaration order, inline flattening, collision diagnostics,
and sensitivity propagation. Map projection retains deterministic key ordering.
Scalar conversion retains the existing number, duration, text-codec, and secret
reference representations.
Depth and node accounting retain their existing meaning, including flattened
containers. Cycle tracking stays active through recursive child traversal and
is balanced on error paths as well as success. Passing state through a helper
does not introduce another full-tree projection or an unnecessary ownership copy.
Cloning and source-value construction remain distinct operations with their
existing semantics. Reuse small helpers when the semantics coincide; keep the
projection refactor centered on making the existing traversal readable.
## Behavior And Scope Preservation
The implementation preserves:
- Root-owned public types, signatures, supported import paths, and facade boundaries.
- Supported source forms, opt-ins, import controls, placement, acquisition and
composition ordering, and source/version evidence.
- Strict peer composition, precedence operations, identity agreement and lifetime,
record ownership, and destination-local inheritance.
- The load lifecycle, including typed defaults, post-decode CLI assignment,
recorded application resolution, and final validation.
- Explicit presence, provenance, history, sensitivity, resource contexts, and
detached publication/query results.
- Established diagnostic codes, semantic paths, source coordinates, ordering,
resource lifetime, and existing limit behavior.
The owning contracts define the extent of these obligations. Incidental private
call sequences, exact helper names, and allocation counts do not acquire new
guarantees through this roadmap. The work adds no consumer options, configuration
syntax, dependencies, or complexity-based gates.
## Relationship To Remediation And Other Work
The completed remediation record at `ea46001:docs/roadmap/remediation.md`
documents the identified correctness and algorithmic efficiency findings.
Several overlap these files, particularly item-scope application matching and
inheritance policy localization.
Use the documented contract and completed remediation as the behavioral baseline.
Keep corrective changes reviewable under their findings, and retain their
regressions through this refactor. The later implementation plan should coordinate
overlapping work and select its starting revision rather than duplicating the
remediation plan here. These structural changes do not close a finding merely
because its function moved or became smaller.
[Configuration discovery](../config.md#configuration-discovery) is implemented.
Import cleanup should preserve its source-acquisition boundary and behavior as
part of this refactor.
## Work Needed And Validation
Implementation needs to consolidate import inputs and adapters, separate import
responsibilities, extract inheritance operations with explicit outputs, and
decompose projection handling. Remove superseded private plumbing as its callers
migrate. Review the resulting call flow for repeated traversals, policy scans,
copies, or hidden state transitions introduced by extraction.
Reuse the existing behavioral suites under the
[development guide](../development.md) and [testing policy](../policy/testing.md):
- Imports: selection, scheduling, source/version evidence, placement, helper scopes,
ordinary/layered equivalence, filesystem failures, and resource lifetime.
- Inheritance: mapping and record relationships, nested destination resolution,
cancellation and replacement, policy ownership, history, sensitivity, and
independent loads.
- Projection: schema/decoding agreement, inline fields, text codecs, nil/empty
values, cycles and budgets, secrets, and detached results.
Add focused regressions only where a consequential handoff risk lacks coverage.
Tests should continue to describe behavior as private functions and files move.
Use the existing everyday check and applicable ownership race checks during
implementation. Complexity measurements can inform review; acceptance does not
depend on numerical scores or a new benchmark campaign.
Update the component overview and import internals to describe the implemented
ownership and file responsibilities when the refactor lands. Update contributor
navigation and test-owner references where moves affect them. Public behavior
continues to be defined by its current canonical owners.
## Completion Criteria
- Import callers pass cohesive named inputs, with clearly owned load-wide and
scope-local state and fewer redundant forwarding variants.
- Import acquisition, item assembly, and evidence projection are easy to locate
and follow across focused private files.
- Inheritance orchestration exposes value, policy, evidence, and accounting work
without requiring readers to reconstruct their interaction from one large body.
- Projection dispatch and substantial value-kind handlers have clear boundaries
and share the original operation state correctly.
- Existing APIs and contracted behavior, including completed remediation, remain
covered by the relevant behavioral and ownership checks.
- Internal documentation matches the resulting implementation.

View File

@@ -6,112 +6,19 @@ commitments. Adopting a candidate requires an explicit design decision and a
separate plan. Significant architectural exclusions and their rationale belong
in ADRs; this document does not maintain a list of rejected proposals.
## 1. Identified record sequences
## 1. Recursive body merging for identified records
Status: deferred. PromptKit proposed this capability from its retired proposal 2,
**Namespaced Collections As Identified Records**. The original proposal is in the
PromptKit repository at
`b5866dd:docs/roadmap/configuration-collections-identified-records.md`.
PromptKit selected mounted mappings instead; this feature is not a dependency
of that design or a request to restore multiple installed prompt versions.
Status: deferred. The implemented
[collection policy](../config.md#strict-document-composition) uses `upsert` to
replace a complete matched record. Recursive merging of fields inside matching
records remains a separate candidate and would revisit the whole-record update
boundary selected by [ADR 0017](../adr/0017-collection-sequence-policy.md).
A narrower [sequence placement and composition feature](../config.md#strict-document-composition)
is implemented: consumers can place imported mappings as sequence items and
register a single literal string identity field for duplicate-rejecting strict
composition. Precedence still replaces sequences as whole values. This entry
tracks broader identified-record behavior that remains deferred and is not part
of that implemented contract.
The active [sequence roadmap](sequences.md) now selects a narrower design for
collection policy, compound scalar identities, final application-registered
identity validation, and inheritance within identified sequences. It owns those
selected behaviors and their remaining decisions. The original broader proposal
below remains background for further consideration, particularly recursive
combination of matching record bodies; it does not override the active roadmap.
### Motivation and proposed behavior
Allow applications to declare that selected sequences contain records identified
by explicit fields, while retaining sequence-shaped output for slice decoding
and effective YAML. This could make composite identities and independently
authored catalogs more convenient than encoding identity in nested mapping keys.
Current [composition](../config.md#strict-document-composition) appends distinct
records only at explicitly registered paths; other sequences remain atomic.
Selected [overlays](../config.md#selected-overlays) always treat sequences
atomically.
The proposed broader opt-in registration would extend the current reusable
application definition from one identity field to a composite identity made of
an ordered, nonempty set of distinct direct string-field names. For example,
records at `profiles` could be identified by `(namespace, id)`. Records
`(provider, brief)`
and `(application, brief)` would coexist, and a later layer could update the
provider record's temperature while retaining its backend and model.
The requested semantics, all still subject to design review, are:
- Match exact string tuples, preserving component boundaries and spelling,
including empty strings. Every contribution supplies its complete identity;
defaults, inheritance, filenames, and normalization do not supply it.
- Retain distinct identities. Matching records from strict peers recursively
compose disjoint body fields and conflict on overlapping non-mapping body
values, even equal values. Equal identity fields act as structural keys rather
than conflicts and retain their contributing source evidence.
- Higher layers recursively overlay matching bodies. A changed identity denotes
a different record, not a rename. Proposed ordering is first contribution in
precedence-assembly order; matching records keep position and new ones append.
- Reject malformed identities, non-mapping records, non-null non-sequence values
at registered paths, and duplicate tuples within one source sequence. Later
layers cannot repair malformed structural identity in an acquired contribution.
- Treat an empty registered sequence as contributing no records; whole-path null
retains ordinary conflict/replacement semantics. Unregistered sequences,
including nested message lists, retain existing atomic behavior.
- Support record inheritance within the same registered sequence using an
`extends` mapping containing the complete parent tuple. Resolve only final
active relationships after precedence, preserve the child's identity, overlay
its body on the parent's body, and consume the directive before decoding.
Preserve cancellation, missing-parent/cycle errors, and declaring resource
contexts. Existing string-based mapping-sibling inheritance stays unchanged.
Integrate ordinary and layered loads, imports, mounted paths, selected overlays,
and whole-sequence CLI overrides. Keep origins, presence, history, sensitivity,
and resource contexts attached to the correct record despite different source
and final indexes. Diagnostics must retain source locations without exposing
sensitive identity values. Final inspection may use snapshot-specific sequence
indexes; existing atomic sequence diffs need not become identity-aware.
Successful results would have valid, unique identities, including after defaults
and application resolution, without a second merge or inheritance pass. Existing
source-version checks remain independent; consumed headers cannot supply missing
record identities. No identity-selector path syntax, arbitrary merge callbacks,
wildcard registration, per-record deletion/rename operator, multiple inheritance,
or cross-path parent references are requested.
### Questions to resolve before implementation
- **Need and delivery scope:** does an active consumer benefit enough to justify
the added semantics over keyed mappings? Are composite identities required?
Staging composition before inheritance is possible but would leave part of
the request incomplete.
- **Defaults and callbacks:** do default records merge by identity or retain
existing sequence-replacement behavior? How are callback-created duplicates,
identity edits, and reordering validated and reflected in provenance?
- **Null, empty, and order:** what happens when null clears a collection and a
later layer reintroduces records? How do absent and explicitly empty sequences
interact with defaults and first-contribution order?
- **Inheritance references:** how do peers compose relationship declarations,
and how are complete references enforced without assembling partial tuples
accidentally? What happens if identity fields overlap structural controls?
- **Registration paths:** how do overlapping registrations and paths beneath
sequence indexes behave when assembly changes those indexes?
- **Evidence and lifecycle:** how are record identities mapped between source
trees, history, inheritance, callback snapshots, and final indexes without
mixing records' provenance or sensitivity? Where do identity checks run relative
to header consumption and other structural processing?
Keep the feature deferred until its consumer value and full lifecycle design
justify reconsidering the sequence-merging exclusion. A correct final sequence
alone would not establish correct inspection or disclosure behavior.
Consider this only in response to a concrete consumer need. A proposal would need
to define conflicts for overlapping body fields, removal of omitted fields,
presence and ordering, and how provenance, history, sensitivity, and resource
context follow partially retained records. Any change requires a new decision and
implementation plan; this entry does not alter current upsert behavior.
## 2. Required version declarations for selected sources

View File

@@ -1,894 +0,0 @@
# Sequence Identity And Combination Implementation Plan
Status: ready for implementation; no stages completed.
Audience: a Luna coding agent implementing one numbered stage per prompt, in
order. This plan starts at commit
`4f941f8ba4c9a9ed0afa87b07fe6ba29f131d1c6`. It implements the settled
[sequence roadmap](sequences.md), [ADR 0017](../adr/0017-collection-sequence-policy.md),
and [ADR 0018](../adr/0018-cli-after-structural-resolution.md).
## Working Rules
- Read [development guidance](../development.md) and the applicable policies,
especially [agents](../policy/agents.md), [architecture](../policy/architecture.md),
[testing](../policy/testing.md), and [documentation](../policy/documentation.md).
Prefer the repository knowledge graph for code discovery.
- The roadmap owns the selected feature semantics. This plan supplies execution
order and implementation choices, not additional product requirements. Preserve
unchanged source, parser, version, schema, ownership, and inspection contracts.
- Complete each stage's implementation and behavioral tests together. Record its
completion and validation here, then commit the nonempty unit with a short,
plain-English message. Preserve unrelated changes. Do not publish intermediate
stages or a release, update consumers' dependencies, or wait for CI.
- Run focused tests while iterating and `./scripts/check.sh` before handing off
each code stage. Use the installed compatible Go toolchain with dependencies
already acquired, as the development guide describes. Add race coverage at the
ownership milestones below. These are implementation checks, not new release
gates. Reuse successful checks for unchanged content rather than repeating them
at a documentation-only milestone. Do not require human platform validation.
- Update affected declaration comments and canonical current-behavior prose when
behavior lands. Keep incomplete target behavior in this roadmap/plan until it
works. The final documentation stage reconciles the cross-feature narrative;
it is not permission to leave misleading current contracts in earlier stages.
- Transitional internal adapters are acceptable where specified. Remove them by
completion. Do not add public compatibility aliases, permanent feature flags,
alternate semantics, new dependencies, new resource caps, or an extension/plugin
framework for this work.
- Preparation stages keep existing public behavior working and test new machinery
privately. Activation stages integrate values, policy, metadata, and relevant
failure handling together; do not defer correct inspection to a later stage.
## Implementation Decisions
### Public Registration Surface
Replace `SequenceComposition` and `Definition.SequenceCompositions` with:
```go
type SequenceIdentity struct { /* private, root-owned registration state */ }
type SequenceSelector []SequenceSelectorSegment
type SequenceSelectorSegment struct { /* private literal-or-any-mapping-key state */ }
func SequenceLiteralKey(key string) SequenceSelectorSegment
func SequenceAnyMappingKey() SequenceSelectorSegment
func SequenceIdentityAt(path Path, fields ...string) SequenceIdentity
func SequenceIdentityMatching(selector SequenceSelector, fields ...string) SequenceIdentity
// Field on Definition[T]:
// SequenceIdentities []SequenceIdentity
```
For example, the reusable definition can contain both registrations:
```go
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
},
```
The exact constructor converts mapping-only `Path` segments into literal selector
segments. Preserve invalid index segments long enough to report an ordinary
definition error; do not panic, silently drop them, or change `Path` semantics.
The zero selector is the exact root selector, harmless under the existing
mapping-root schema. A zero selector segment denotes the literal empty key,
not a wildcard. Empty field names are valid literal field names; an empty field
set is invalid. Constructors snapshot slices; load-entry conversion also takes
an owned snapshot. Definition validation, not construction, reports errors.
Root types and functions must not expose internal types or bridge methods.
### Identity Representation And Matching
- Normalize identity field sets once: deduplicate and sort exact field names.
Use the same normalizer for YAML and application registration.
- Encode a record's identity as a hashable, type-tagged, length-delimited tuple in
normalized field order. Mapping lookup indexes avoid scanning record fields
repeatedly. Distinguish structural-payload lookup from final typed-payload and
parent-reference lookup: only fields acting as directives are excluded, not
every occurrence of the same literal spelling. Never include tuple encodings
in diagnostics.
- Normalize finite JSON numbers to sign, significant decimal digits, and a signed
base-10 exponent. Strip insignificant zeroes and give all zero spellings one
representation. Use `math/big.Int` for exponent arithmetic, not a power-of-ten
expansion or a float conversion. Thus large exponent spellings do not allocate
storage proportional to their mathematical magnitude. This comparison-only
representation does not rewrite document nodes or typed projection.
- Compile selectors into a mapping-segment trie with literal and wildcard edges.
Carry applicable trie states through traversal; stop at sequence-index edges.
Index concrete matches within one assembly/inheritance scope where useful, but
do not save an assembly-only destination list for final validation.
- Check registration overlap pairwise once during definition validation using
equal length and compatible corresponding segments. This straightforward
`O(R² × D)` check, for R registrations and maximum selector depth D, is
proportionate for registration definitions. Equivalent constraints coalesce;
conflicting overlaps fail even with no matching input.
Do not impose a registration-count limit to simplify this operation.
### Policy And Evidence Storage
Keep application constraints separate from YAML policy. Add private engine
collection-policy records containing normalized identity, explicit mode, and
declaration locations. Track whether identity is pending or belongs to a retained
collection. Retain declaration presence separately from inferred defaults.
Use the existing acquisition/item scopes as ownership anchors, with an opaque
per-load owner token when a retained record needs one. Store item-local policies
relative to their owning record, and project concrete paths when that record is
placed. Do not key ownership solely by numeric index, identity tuple, or source
filename: all three can coincide for a replacement that must lose old policies.
Inheritance clones the relevant scoped state; it does not create registrations.
Resolve policies through shared engine helpers used by strict composition,
precedence, and inheritance. Do not attach engine policy to public values or add
policy fields to the parser-independent `document.Node`. The sidecar can be
extended with concrete-path indexes; it need not become a general event store.
Sequence combination returns the resulting list plus explicit input-to-output
item projections and retained/replaced contribution information. Use that same
result for values, policy transfer, origins, source destinations, sensitivity,
and historical replay. Extend the existing sequence-history descriptors for
noncontiguous upsert positions rather than inferring them from equal values.
Sharing immutable internal evidence is acceptable under existing ownership rules;
independently returned mutable records must remain detached.
### Diagnostics
Use existing `options.invalid` for invalid application definitions and existing
`compose.invalid` / `compose.conflict` for malformed or duplicate strict-peer
contributions. Add root-owned and engine diagnostic constants
`CodeSequenceInvalid` (`sequence.invalid`) and `CodeSequenceConflict`
(`sequence.conflict`) for malformed collection declarations/operations or final
identities, and conflicting policy / precedence duplicates respectively.
Final duplicate identities use `sequence.conflict`. Keep existing inheritance
invalid/missing/cycle codes for record relationships. Use existing override and
schema/decode codes at the CLI boundary; do not invent a new error taxonomy there.
Report collection/item paths and available locations, not identity values,
normalized tuple strings, raw parent references, or raw hook errors. Declaration
conflicts include both sites. Definition overlap errors identify registration
positions without fabricating a concrete matched configuration path. Exact
English diagnostic wording is not a new guarantee.
## Stage 1 — Implement Shared Scalar Identity Primitives
Scope: new private identity helpers and their focused tests; no load-path change.
1. Add `internal/engine/sequence_identity.go`. Implement field-set normalization,
scalar tuple construction, and exact decimal normalization as chosen above.
Accept string, number, boolean, and explicit null nodes; distinguish categories
and absence. Exclude fields acting as structural directives from payload lookup.
2. Validate mapping records, complete direct-field sets, and uniqueness using one
tuple-to-first-record index. Return paths and source locations, not values.
Support caller-selected diagnostic families for strict and other boundaries.
3. Cover equivalent field sets; empty strings and null; missing/non-scalar fields;
literal punctuation; compound tuple boundaries; `1`/`1.0`/`1e0`; signed zero;
close unequal decimals; and large exponents without expansion. Verify that
node text remains unchanged. Include `extends: null` as a structural field
that cannot supply source identity.
Validation: `go test ./internal/engine -run 'SequenceIdentity|SequenceNumber'`
and the everyday check. Retain the old registration path until its adapters are
replaced; do not generalize unrelated YAML scalar decoding.
## Stage 2 — Compile Exact And Dynamic Application Constraints
Scope: private registration model, selector matching, overlap validation.
1. Extend `sequence_registration.go` with the common registration representation
and compiled selector trie. Represent literal and arbitrary-mapping-key segments
structurally, not with a magic string. Snapshot registration inputs.
2. Validate field sets and mapping-only selectors before source acquisition.
Preserve existing destination validation without adding restrictions on identity
field spelling. Root and unused selectors remain harmless.
3. Implement full-destination matching, equivalent overlap coalescing, and
incompatible-overlap errors. Exact rules have no priority over dynamic rules.
Keep a temporary conversion from the current single-field registration so
existing callers still use their current behavior.
4. Test exact and wildcard matches, multiple wildcard levels, empty/punctuation
keys, sequence-index nonmatches, disjoint lengths/literals, equivalent sets,
incompatible overlaps with absent input, and independent uniqueness scopes.
Validation: focused registration/selector tests and the everyday check. This
stage does not export the replacement API or add final-result validation yet.
## Stage 3 — Parse Collection Declarations Without Filesystem Coupling
Scope: private parser and directive extraction helpers, initially unit-tested
without activating collection combination in public loads. Keep source-form
activation out of this stage; ordinary sources and selected overlays are wired
in Stages 13 and 14 respectively.
1. Add `sequence_declaration.go` for source-root `configkit.sequences`. Each entry
has a required existing literal `destination` form and optional `key` and
`mode`. `key` is a string or nonempty string list; `mode` is exactly replace,
append, or upsert. Use shared identity normalization. A destination-only entry
can be a harmless no-op; do not add an unrequested ban on empty declarations.
2. Split common root-directive extraction from import-specific parsing in
`import_spec.go` / `parse.go`. Extract imports and sequences together, retaining
source-local locations and explicit presence. Reject unrecognized directive
fields and wrong supplied field types using existing strict parsing style.
3. Keep declaration destinations relative to source placement, with item-local
coordinates retained until their scope is assembled. Do not project them
prematurely to a guessed numeric index.
4. Give extraction an explicit overlay context: sequences are allowed, imports
are not. Import enablement remains separately enforced; do not turn on
filesystem access because a source declares sequences. Keep enabled
source-version checks in raw source coordinates before consuming controls;
extend existing actual header/control-overlap handling to sequence directives
without treating an absent header beneath a control as an actual overlap.
5. Test key-only, mode-only, combined, repeated-field-set equivalence, no-op,
malformed types/modes/destinations, mixed imports/sequences, and overlay rules.
Do not add import-level key/mode shorthand or sequence wrappers.
Validation: focused declaration/parser tests, existing import-spec tests, and the
everyday check. Existing import-only behavior remains intact during preparation.
## Stage 4 — Reconcile Declarations And Select Collection Operations
Scope: pure agreement and operation-selection helpers, independent of loading
and of the ownership transformations introduced next.
1. Add `collection_policy.go`. Gather the current scope's declarations before
combining peers. Reconcile equivalent keys and agreeing explicit modes;
conflicting peers fail. Omitted fields have no preference. Apply authoritative
application constraints even for key-only declarations without a value.
2. Implement shared operation selection: current explicit mode, retained explicit
mode, inferred upsert with effective identity, otherwise replace. Persist only
explicit modes. Policy-only declarations must not replay earlier combinations.
3. Accept eligible lower policy as an input; do not decide record survival or
ancestor removal inside declaration reconciliation. Retain declaration
locations/presence separately from inferred mode and application constraints.
4. Test equivalent/different field sets, agreeing/conflicting peer modes,
omission, explicit default modes, and operation inference. Include key-only
declarations agreeing or conflicting with application constraints before a
sequence exists.
Validation: focused policy-agreement/selection tests and the everyday check.
Do not activate collection combination or implement the ownership layer here.
## Stage 5 — Track Policy Lifetime And Record Ownership
Scope: private scoped-state transformations built on Stage 4's resolution helpers.
1. Implement key agreement across retained values, replacement reset, pending
key declarations, and immediate validation when a key-only declaration
identifies retained records.
Changing mode to replace without a replacement value does not reset identity.
2. Implement scope/record ownership transfer and retirement hooks. Actual ancestor
replacement removes descendant collection identity and removed-record policy;
independently owned modes and application constraints survive as specified in
the roadmap. Do not repeatedly reapply old declarations and resurrect reset
identity. Pending declarations retain their owning-scope rules.
3. Use item/acquisition scope anchors and record-owned relative policy as selected
above. Provide transformations for retention, whole-record replacement,
whole-sequence replacement, and inheritance copies without wiring them into
public loading yet. Application registrations remain destination constraints.
4. Use table-driven tests derived from the roadmap's lifetime, persistence, and
replacement tables, including absent/non-sequence destinations, equal outer
identities with distinct record owners, and independent root versus item policy.
Validation: focused collection-policy tests and the everyday check. This stage
must not add path-existence, shape, nullability, or global YAML final-check rules.
## Stage 6 — Implement The Shared Sequence Operations
Scope: a pure document-level combination helper, not separate implementations
for overlays and inheritance.
1. Add `sequence_combine.go`, accepting lower/higher sequence nodes, resolved
policy, and combination context. Reuse Stage 1 identity extraction and indexes.
2. Implement replace, duplicate-rejecting append, and whole-record upsert. Upsert
retains matched lower positions and appends new higher identities in order.
Return item projections and replacement/retention evidence with the new value.
3. Keep peer validation distinct: duplicate identities within any contribution or
among peers fail under every mode. Do not merge record payloads or sort by ID.
4. Cover absent versus null/non-sequence lower values; higher omission, `[]`, null,
and non-sequence values; missing identity under append/upsert; empty inputs;
stable ordering; equivalent/different keys; and explicit replacement identity
reset. An empty input does not bypass kind checks or identity requirements.
5. Verify clones do not alias mutable input nodes or policy slices. Build indexes
once per operation rather than scanning the accumulated list for every record.
Validation: focused combination tests and the everyday check. Do not change the
ordinary recursive mapping rule or add unkeyed append.
## Stage 7 — Extend Sequence History Representation And Replay
Scope: represent and replay sequence operations without yet changing all metadata
projection consumers or public precedence.
1. Extend `sequenceHistoryContribution`, sequence replay in `history.go`, and
its contribution indexes to consume Stage 6's explicit item mappings. Existing
append ranges may remain a compact case; upsert cannot be represented by one
offset.
2. Preserve authored raw contribution snapshots and displaced values. A replaced
record's omitted fields must appear as removed, not as retained lower payload.
Preserve equal-valued replacement events. Keep existing index-based query and
atomic sequence-diff APIs; add no public identity-selector path syntax.
3. Exercise synthetic append/upsert/replace histories, noncontiguous matches,
nested paths, and discarded descendants. Compare replayed final values with
the actual combination result, including first/last and unchanged neighboring
records. Reuse existing escaped-path and detached-query tests rather than
duplicating them.
Validation: sequence/history/replay tests and the everyday check. Keep current
append/replace callers working through the extended descriptors.
## Stage 8 — Project Sequence Origins And Inspection Evidence
Scope: use the prepared item mappings consistently in metadata consumers.
1. Project composition origins, nested source destinations, historical paths, and
sensitivity through Stage 6's item mapping and Stage 7's history descriptors.
Physical source IDs/locations and resource contexts continue to identify the
declaring source. Reuse one projection mechanism rather than searching again
by identity or equal payload.
2. Test synthetic noncontiguous upserts, replaced descendants, nested source
destinations, unchanged neighbors, and secret-marked replaced records. Verify
projected queries agree with replayed and actual final values, retaining
displaced evidence and detached results.
3. Keep the new precedence operations unactivated until their value, policy, and
projection machinery are integrated in the later load-path stages.
Validation: metadata/history/sequence/provenance/sensitivity tests,
`go test -race ./internal/engine`, and the everyday check.
## Stage 9 — Separate Per-Input Acquisition From Composition
Scope: a behavior-preserving acquired-input handoff; grouped underlay coordination
is the next stage.
1. In `load_input.go`, `imports.go`, and related graph helpers, retain acquisition
scopes, raw contributions, inclusion edges, source records,
contexts, and extracted declarations without irrevocably composing them.
Reuse existing `deferAssembly` and acquired-scope mechanisms instead of adding
a second loader. Keep already acquired document handles closed as before.
2. Separate acquisition from the per-input composition entry. Retain pending
item/helper scope boundaries, namespace identities, inclusion/repetition checks,
source records, accounting, and scheduler inputs. Do not reacquire sources to
compose them later or change grouped coordination yet.
3. Preserve existing version checks for this refactor. Retain a small adapter
invoking current composition semantics at the existing lifecycle point.
Preserve source-version callbacks at acquisition, including header consumption.
4. Run existing acquisition, dependency scheduling, item placement, source-version,
ordinary/layered equivalence, resource-lifetime, and source-context tests. Add
only targeted regressions for any newly split handoff not already covered.
Validation: the focused suites above and the everyday check. This refactor must
not change source IDs or turn acquisition order into precedence order.
## Stage 10 — Separate Underlay-Group Acquisition From Composition
Scope: behavior-preserving group coordination in `load_request.go`, using the
per-input handoff already prepared in Stage 9.
1. Separate acquiring an underlay group from scheduling/composing its acquired
scopes. Preserve namespace reanchoring, shared inclusion/repetition state,
item-local helper scopes, source/context aggregation, and group accounting.
2. Keep authored acquisition order: main and depth-first imports, each underlay
and input, overlay, then synthetic CLI records. Preserve underlay indexes and
current version-check timing in this refactor; policy-dependent layered
composition is activated only in Stage 15.
3. Expose the common scope-composition handoff with inputs for accumulated lower
policy and compiled application constraints. Initially adapt it to the current
semantics. Do not add another loader or repeat acquisition/version callbacks.
4. Test the split with existing grouped/dependency/version/destination coverage,
including empty inputs/groups and repeated sources within versus across tiers.
Confirm unchanged values, source ordering, contexts, and failure evidence.
Validation: grouped/layered, import scheduling, version, destination, and
resource-context tests plus the everyday check.
## Stage 11 — Compose Strict Collections With Resolved Identity
Scope: the policy-aware strict composition core. Nested pending-item scope
integration is deferred to Stage 12, not absorbed into this prompt.
1. Extend `composeDocumentsWithPolicy` and `validateRegisteredSequences` to use
normalized field sets and scalar tuples. Resolve current declarations,
applicable application constraints, and eligible lower identity before peer
composition, using the Stage 4 agreement and Stage 5 lifetime helpers.
2. Use the existing dependency-ready order for contributions and authored list
order. Validate every identified sequence, including singleton contributions;
strictly append distinct peers and reject duplicates under all modes.
3. For explicit replacement, do not use obsolete lower YAML identity to authorize
otherwise unkeyed peer composition. Still apply independently registered
application constraints and genuinely applicable declarations.
4. Attach Stages 7–8 evidence and Stage 5 scoped policy to composed results. Test
completed-document contributions with lower-established identity, declaration
conflicts, compound/scalar IDs, singleton validation, and replacement resets.
Validation: composition and sequence identity tests plus the everyday check.
Keep unregistered non-mapping conflicts unchanged. Public load activation remains
in the later integration stages.
## Stage 12 — Integrate Policy-Aware Item And Helper Scopes
Scope: adapt pending-item/helper composition to Stage 11's core using the acquired
scope boundaries from Stages 9–10.
1. Gather declarations before composing each relevant scope, complete item-local
helper imports before item placement, and consume applicable non-main version
headers before checking record payload identity. Keep relative policy with its
owner until concrete placement is known.
2. Apply eligible lower identity at each scope, but prevent a replacement outer
record from borrowing nested identity owned by the discarded lower record.
Preserve independently applicable policy and application constraints.
3. Retained structural directives cannot serve as identity fields. A helper may
supply payload identity components. Do not require a helper-built compound
parent reference to be complete at this point; validate the completed surviving
reference during inheritance.
4. Project item evidence and owned policies through Stages 7–8 machinery. Test
nested items, helper-supplied identity, header consumption, declaration scope,
discarded owner identity, and exact diagnostic paths with no value disclosure.
Validation: item/destination/import/composition/source-version tests and the
everyday check. Keep current public behavior until load-path activation.
## Stage 13 — Activate Ordinary Source Collection Declarations
Scope: ordinary `Load` base-source assembly, excluding activation of the new
selected-overlay policy surface until Stage 14.
1. Wire common directive extraction into every ordinary source form: file,
native filesystem, bytes, reader, mapping, imported mapping, and sequence item.
Declarations work without `Imports`; disabled imports retain their existing
rejection semantics. `NoSource` remains absence rather than an empty document.
2. Use Stages 11–12 composition and Stage 5 scoped policy. Strip collection
controls from application decoding, but retain their locations/evidence for
their structural lifetime and diagnostics.
3. Integrate policy/projections together with the values. Cover inline/imported
equivalence, placement-relative declarations, singleton and peer validation,
and declarations without accompanying values. Maintain ordinary loading's
existing non-collection overlay behavior while preparing its policy handoff.
4. Update the implemented declaration/composition contract sections. Collection
precedence, inheritance, and later CLI behavior remain incomplete until their
integration stages; do not claim them implemented here.
Validation: ordinary-load, import, composition, and metadata tests plus
the everyday check. Include one public ordinary-load declaration example test.
## Stage 14 — Activate Selected Overlay Collection Policies
Scope: ordinary loading's selected overlay and shared precedence integration,
using the source/declaration handoff already active in Stage 13.
1. Permit `configkit.sequences` in selected overlays while continuing to reject
overlay imports. Preserve source-version header/control handling and ordinary
overlay namespace selection. Consume controls before application decoding.
2. Use Stage 6's sequence operations and Stages 4–5 policy helpers in the assembly
accumulator. Mappings still recurse and unrelated sequences still replace.
Wire value, policy retirement/transfer, and Stages 7–8 inspection projections
together, not as a later best-effort repair pass.
3. Cover append/upsert/replace, empty/null/kind cases, key-only/mode-only changes,
retained explicit modes, overlay identity agreement, and independently owned
policies. Check values, history, origins, and discarded descendants together.
4. Update the selected-overlay and collection-operation contract sections. Make
the shared precedence entry reusable by layered loading without adding another
operation implementation or turning overlay imports on.
Validation: ordinary-load, overlay, composition, policy, and metadata tests plus
the everyday check.
## Stage 15 — Integrate Lower Policy Across Layered Assembly
Scope: source-backed underlay coordination and the layered selected-overlay path,
using the prepared acquisition, ownership, strict-scope, and precedence machinery.
Do not absorb unfinished work from Stages 5–14 into this stage.
1. Acquire main first, but assemble underlays from lowest to highest, strictly
composing each group with applicable accumulated lower policy before applying
its one precedence operation. Then compose/apply main and the selected overlay.
Do not sequentially upsert peer sources as if they were different tiers.
2. Pass lower identity into higher strict scopes before peer composition. Cover an
otherwise empty underlay establishing identity, inherited explicit mode, and
replacement that deliberately discards the old YAML key. Invalid lower tiers
still fail even if a higher replacement would hide their values.
3. Keep main-only version ownership and every enabled per-source version rule.
In default source-version mode, run the main checker once on the composed
main tier before overlay/inheritance/defaults, never on underlay-augmented
values. Its former main-before-underlay composition timing need not persist:
main composition now depends on lower policy. Enabled-mode source checks keep
acquisition order and exact source-local header handling. No extra callbacks
or second acquisition pass are permitted.
4. Keep histories in precedence order, sources in acquisition order, underlay
indexes stable across empty groups, and shared budgets intact. Update the
layered contract's acquisition-versus-composition description accordingly.
Remove the acquisition/composition adapters superseded by this integration;
the old public-registration adapter remains until Stage 22.
5. Cover multiple inputs/groups, mounted main inputs, item scopes, empty main,
explicit empty groups, layered overlays, version protection and diagnostics,
and equivalence with ordinary loading where the inputs express the same model.
Validation: layered/grouped, source-version, dependency scheduling, destination,
and metadata tests plus the everyday check.
## Stage 16 — Verify Replacement And Nested Policy Ownership End To End
Scope: the consequential lifetime cases across the now-active source assembly.
1. Exercise an item-local nested collection through retaining the outer record,
upserting the same outer ID, and replacing the entire outer sequence. Retained
items keep their local key/mode; replacements do not inherit discarded policy
merely because identity, source label, or final index is equal.
2. Exercise ancestor replacement and later reintroduction. Removed collection
identity does not reappear; independently declared append mode can survive
and require renewed identity. Cover renewed YAML key, application constraint,
explicit replace, and the intended failure with no available identity.
3. Cover key-only validation of retained records, pending declarations, a no-value
replace declaration that cannot authorize key changes, and independent policy
whose owning scope survives a record replacement.
4. Correct any integration defects in the Stage 5 ownership transformations;
consolidate tests with their mechanism owners. Do not introduce special-case
ownership by numeric path or a historical identity cache to make cases pass.
5. Verify frozen metadata, origin/resource context, sensitivity of discarded
records, and mutable independence across repeated loads for these scenarios.
Validation: focused policy/sequence/layered/ownership tests,
`go test -race ./internal/engine`, and the everyday check.
## Stage 17 — Prepare Inheritance Policy And Template Transport
Scope: behavior-preserving inheritance state transport, before activating
collection-aware inheritance operations in Stage 18.
1. Extend inheritance templates/expansion sidecars to carry collection identity
and record-local policy alongside nodes and origins. Keep existing scheduler
keys, cancellation, version protection, missing-parent/cycle diagnostics, and
destination-local evaluation of nested templates.
2. Provide owned-state copy/transfer helpers from Stage 5 and path projections from
Stage 8. Preserve original nested instructions for each destination's later
evaluation rather than retaining only a parent's expanded output. Keep
application registration matching separate from copied YAML policy.
3. Test template reuse at multiple destinations, independent scoped state, and
retained instruction locations/resource context. Preserve current inheritance
values/history while these new helpers remain preparatory; do not activate a
value change without its corresponding policy/evidence handling.
Validation: existing inheritance, ownership, history, and resource-context tests,
focused transport tests, and the everyday check.
## Stage 18 — Apply Collection Policy During Mapping-Sibling Inheritance
Scope: activate shared collection operations using the prepared template/state
handoff; existing mapping-sibling relationship selection remains unchanged.
1. Use the shared sequence combiner at the inheriting destination: parent lower,
child higher, destination explicit mode or identity-based default. Copy YAML
identity where applicable, but do not automatically copy a parent collection's
explicit mode to a different destination. Match application constraints at the
destination, including newly created matching paths.
2. Transfer record-owned state only with surviving records using Stage 17's
helpers. Integrate inheritance evidence with Stage 8's projections in the same
activation, preserving origins, historical sensitivity, resource context,
cancellation, and existing expansion/history accounting.
3. Cover nested append/upsert/replace, explicit empty/null/kind cases, copied key
with inferred upsert, destination explicit mode, incompatible keys, application
constraints, cancellation, and policy at an inherited new namespace.
4. Update current mapping-sibling inheritance documentation for collection
operations. Identified-record parent lookup remains incomplete until Stage 21.
Validation: inheritance, history, provenance, sensitivity, and resource-context
tests plus the everyday check. No identified-record parent lookup is active yet.
## Stage 19 — Build Identified-Record Inheritance Graphs
Scope: relationship extraction/validation and shared scheduling for sequence
scopes; test these helpers before activating payload expansion.
1. Extend sibling-scope discovery to identified sequences. Build the tuple-to-item
lookup once using effective identity. Every record supplies its own complete
identity before inheritance; neither parent lookup nor typed defaults supplies
a missing child field.
2. Parse a surviving record's `extends` as null cancellation or a mapping with
exactly the normalized identity fields and supported scalar components.
Mapping-sibling references keep their existing string form. Do not accept a
scalar shorthand, multiple parents, positional or cross-collection lookup.
3. Parent-reference mappings have already used ordinary strict mapping
composition: disjoint helper fields can complete one reference, while equal
overlapping fields conflict. Validate completeness/extra fields only now on
the surviving reference. Discarded references remain historical evidence.
4. Build one-parent dependency edges and reuse `scheduleDependencies` with stable
semantic-pointer keys. Missing parents, self-reference, and longer cycles
use existing inheritance diagnostics without identity values. Keep authored
output positions independent from graph schedule.
5. Cover compound and null references, numeric equality, forward edges, chains,
unrelated equal identities in different lists, malformed refs, missing parents,
cycles, helper-built references, and overwritten/cancelled relationships.
Validation: focused inheritance graph/reference tests and the everyday check.
## Stage 20 — Prepare Reusable Inheritance Scope Execution
Scope: remove mapping-container assumptions from the private template executor
without activating identified-record inheritance in public loads.
1. Refactor template execution to separate relationship scheduling/payload
expansion from reading and rebuilding its scope container. Preserve current
mapping-sibling output, locations, cancellation, evidence, and budgets.
2. Provide private mapping- and identified-sequence-scope adapters using Stage 19's
relationship graphs. The sequence adapter preserves output positions and
exposes child identity fields to the payload-expansion step. Keep this private;
do not add a public generic inheritance or merge callback API.
3. Reuse Stage 17's template/policy transport and Stage 18's collection-aware
payload operations. Existing origin/history/retained-node handling must remain
available to both adapters rather than duplicating an entire executor.
4. Run existing mapping-inheritance regressions and privately exercise a small
sequence graph through the adapter. Check output order, independent templates,
and operation/evidence paths before public activation in Stage 21.
Validation: inheritance execution, scheduling, history, and budget tests plus
the everyday check. This refactor does not introduce new parent-reference forms
or change public loading semantics yet.
## Stage 21 — Resolve Record Inheritance With Ownership And Evidence
Scope: integrate Stage 19's graphs and Stage 20's executor into the structural
expansion phase, with values and inspection working together.
1. Expand parent payload into the child while preserving the child's own identity
fields and their origin/presence. Exclude parent identity fields from payload
inheritance. Recursively combine mappings and use destination collection policy
for nested sequences; retain whole-record upsert at the earlier boundary.
2. Consume the child's directive before decoding. Keep parents as ordinary records
subject to the same schema/semantic validation; create no abstract templates or
acquired source records.
3. Resolve only records surviving tier assembly. Cover parent upsert/removal,
replacement of a child without `extends`, self-reference not exposing a lower
predecessor, and nested destination-local relationships. Enclosing scopes
finish before nested scopes; assembled list order stays unchanged.
4. Carry policy, origin/history, sensitivity, resource context, and existing
expansion/retained-history accounting through copies. Do not widen budgets or
require a new inspection-path API.
5. Add public-load regressions for complete inheritance chains and nested
collections, including secret canaries on success and failure. Update the
implemented inheritance contract and consumer guidance for this scope.
Validation: all inheritance and cross-feature inspection tests,
`go test -race ./internal/engine`, and the everyday check.
## Stage 22 — Expose The Root-Owned Registration API
Scope: the settled facade surface and migration of in-repository callers.
1. Implement the exact public names/signatures above in a focused root file, and
add `Definition.SequenceIdentities`. Convert root records/selectors explicitly
into the engine model. Snapshot fields, selectors, and paths on construction
and loading, preserving immutable definition reuse.
2. Remove `SequenceComposition`, `Definition.SequenceCompositions`, and the private
old-shape adapters. Migrate all maintained code/tests/examples; do not edit
historical release notes solely to erase the old name or add deprecated aliases.
3. Cover mixed exact/dynamic constructors, compound scalar fields, literal `*`
versus wildcard segments, root/unused selectors, invalid index paths and empty
field sets, overlap failure before source access, and independent loads.
4. Prove dynamic constraints apply before strict composition and at inherited
destinations, without consumer namespace enumeration. Keep final-result
enforcement described as pending until Stage 27 actually adds it.
5. Verify the public boundary and declaration documentation. Do not expose engine
enums, policy stores, parser nodes, or internal methods.
Validation: root sequence/facade, registration, public/dependency-boundary, and
consumer example tests plus the everyday check.
## Stage 23 — Separate Reusable CLI Override Preparation
Scope: mechanical preparation refactor and a privately tested post-structural
validation entry, without changing current CLI processing order.
1. Refactor override preparation so labels, paths, snapshots, type support,
disjointness, version protection, and existing budgets remain reusable outside
the source assembly accumulator. Allocate synthetic sources once in the same
order; do not reopen or reinterpret CLI values as YAML.
2. Prepare post-structural validation that rejects `extends` as well as
`configkit` in supplied paths and nested values, using their established
structural roles. Test this entry privately; retain the existing entry's
behavior until the lifecycle switches in Stage 26. Do not reject previously
valid early-CLI instructions before that switch or add unrelated restrictions.
3. Test detached prepared batches, synthetic source ordering, and the new
directive validation. Reuse existing label/path/cycle/type-support/budget tests
rather than adding another inventory of the same preparation cases.
Validation: override preparation/source tests and the everyday check. Keep
ordinary/layered behavior unchanged; typed assignment is the next stage.
## Stage 24 — Implement Schema-Aware Post-Decode CLI Assignment
Scope: a private typed assignment helper using Stage 23's prepared overrides,
without activating the later lifecycle order yet.
1. Add `applyDecodedOverrides`, using existing schema/decode mechanisms to bind
only supplied override subtrees against a detached copy of the decoded state.
Mappings recursively overlay, sequences replace, and explicit zero/empty/null
retain normal schema rules. Current typed values supply retained children,
not new defaults for replaced list items. Do not call `Definition.Defaults`
again or re-decode the whole projected configuration as newly authored input.
2. Return the updated typed value, canonical post-CLI projection, presence/origin
changes, and existing raw CLI history contributions. Preserve a pre-CLI typed
projection for decoding reconciliation/history. No resolver runs on a failed
override; no partially applied result is returned.
3. Test unknown/type-invalid fields, mapping retention, sequence replacement,
nested maps/lists, named/text/duration/secret schema fields, explicit null/false/
zero, discarded children, structural controls, and mutable ownership.
Validation: override, decode, schema, projection, and ownership tests plus the
everyday check. Only the helper is added here; the shared lifecycle does not
switch until its inspection/context support is prepared in Stage 25.
## Stage 25 — Prepare Post-Decode CLI Inspection And Effective Context
Scope: prepare history, resolution-input, and secret-lookup context handling;
retain the existing public CLI lifecycle until Stage 26.
1. Extend existing replay/reconciliation to represent the target order:
source tiers, inheritance, typed decoding/defaults, CLI, application resolution.
Keep the structural input, pre-CLI typed snapshot, raw CLI contributions, and
post-CLI projection distinct where needed. Use the existing history machinery,
not a second replay framework, and retain legacy input compatibility until the
lifecycle switch. Do not mislabel CLI changes as defaults or inheritance.
2. Prepare resolution-entry metadata from Stage 24's result so explicit presence,
origins, source records, and resource contexts describe the overridden state.
3. Prepare the effective reference/location handoff for secret lookup and final
validation using the post-override/post-resolution values, not stale structural
input. Preserve existing no-credential-byte and ownership behavior.
4. Test the target pipeline with privately assembled snapshots: equal-value CLI
replacement, replaced descendants, decoding reconciliation before CLI, later
recorded resolver transformations, explicit-presence defaults, and an overridden
secret reference. Verify current public-load regressions still pass unchanged.
Validation: history/metadata/resolution/secret tests and the everyday check.
Public activation must not require a later stage to repair these mechanisms.
## Stage 26 — Activate CLI After Decoding In The Shared Lifecycle
Scope: activate ADR 0018 through both loading paths using Stages 23–25's complete
assignment, inspection, and effective-value handoffs.
1. Remove CLI assignment from ordinary/layered structural assembly. Invoke
Stage 24 after inheritance and successful typed-default creation/decoding,
immediately before application resolution. Activate Stage 23's post-structural
validation. Keep the single shared lifecycle, not separate CLI implementations.
2. Wire Stage 25's history and context handoffs in the same change. Resolution
receives explicit-presence/origin metadata and effective CLI values;
`Resolution.Default` preserves them while recorded transformations can change
them normally. Secret lookup uses effective references and locations. Preserve
exactly-once default-factory behavior and synthetic source ordering.
3. Cover `NoSource`, ordinary/layered equivalence, CLI clearing/replacing an
identified list despite append/upsert mode, derived values seeing CLI input,
equal-value provenance, and resolver changes after CLI. Add explicit regressions
that CLI cannot fix earlier decode/inheritance errors or introduce/cancel a
relationship. Update obsolete early-CLI tests deliberately.
4. Update the lifecycle, override, history, and secret-reference contracts and
affected examples to describe the now-implemented order.
5. Remove the superseded early-CLI application paths, old preparation entry, and
temporary replay compatibility paths. Keep their still-relevant behavioral
tests under the new lifecycle rather than postponing code cleanup to the
documentation stage.
Validation: override/lifecycle/resolution/inheritance/history/secret tests,
`go test -race ./...`, and the everyday check.
## Stage 27 — Enforce Final Application Identity Constraints
Scope: the final-result guarantee, not another combination or inheritance phase.
1. Add shared final identity validation after effective secret lookup and before
application semantic validation/publication in the common lifecycle. Use the
canonical effective typed projection, which retains secret references rather
than credential bytes. Preserve the existing validated publication snapshot
and detachment behavior.
2. Traverse the actual final tree with the compiled selector trie, validating each
matching sequence independently with Stage 1 primitives. This must discover
paths introduced by typed defaults, CLI, and callbacks, not only assembly paths.
Support root/namespaced rules and unrelated unselected sequences.
3. Validate actual sequences only. Missing paths, null, and other shapes remain
the schema/application validator's responsibility. Identity extraction here
examines effective typed payload, not whether a field was authored. Do not
recheck YAML-only policies or every intermediate callback state.
4. Cover duplicate/missing/non-scalar identities produced at each late phase,
valid null/scalar identities, numeric equality after typed conversion, dynamic
namespaces created by each late phase, harmless unmatched rules, concrete-path
diagnostics with available origins, and secret-value non-disclosure.
5. Prove both `Load` and `LoadLayered` use this guarantee and return no usable
result on failure. Update registration comments and canonical contract with
this final boundary, excluding later caller mutations from the guarantee.
Validation: final identity, selector, facade, defaults/resolution/override, and
secret tests plus the everyday check.
## Stage 28 — Exercise Representative Consumer Workflows
Scope: a small number of meaningful integrations, not a Cartesian test matrix.
1. Extend `examples/consumers` with an executable source-backed catalog: underlay
records, main fragments, selected overlay append/upsert/replace, and explicit
record inheritance. Demonstrate complete replacement of a matched record,
stable order, and final inspection without application merge code.
2. Add a namespaced typed-map example using one wildcard registration plus an
exact root registration, with independent uniqueness. Show YAML-declared
identity without registration separately so the final guarantee is not implied.
3. Include one integrated CLI/resolution case and one representative provenance/
resource-context/secret-history case where those interactions add protection
beyond mechanism tests. Keep application domain validation recognizable.
4. Audit coverage against the roadmap's work inventory, especially helper-built
parent references, version-header consumption, pending declarations, lower
identity before higher peers, replacement ownership, and dynamic destinations
introduced through inheritance and late typed changes. Add only missing,
consequential regressions at their narrowest stable owners.
5. Consolidate obsolete/brittle tests. Do not restore old replacement semantics
merely to keep fixtures unchanged, and do not preserve duplicates of the same
semantic table across every layer.
Validation: `go test ./examples/consumers`, focused root integration tests,
`go test -race ./...`, and the everyday check.
## Stage 29 — Reconcile Documentation
Scope: finish the current-contract narrative. Code cleanup belongs with the
integrations that supersede each adapter, not in this documentation prompt.
1. Reconcile `docs/config.md` for declarations, identity equality, selectors,
operations/persistence, ownership, inheritance, final validation, and CLI order.
Keep its operation and presence tables authoritative; Go comments own exported
symbols and consumer guidance links rather than copying full contracts.
2. Update `docs/consumers/README.md`, relevant root examples, `README.md`,
`docs/internal/overview.md`, `docs/internal/imports.md`, and
`docs/development.md` where the implemented structure or reading/test map has
changed. Remove stale claims about string-only registration, universal sequence
replacement, early CLI, and overlay prohibition of all directives.
3. Remove implemented portions from the deferred identified-record discussion in
`future.md`, retaining genuinely deferred recursive body merging and other
still-relevant ideas. Preserve accepted ADR decision content; update only status
or implementation references where appropriate. Architecture already adopts
the decisions: remove pending-implementation wording when true, without adding
new policy. Leave release version selection/publication to the release task.
4. Search maintained prose/examples for obsolete registration names and lifecycle
claims; retain intentional historical references. Follow canonical ownership
rather than copying the complete contract into each guide.
Validation: local documentation-link checks and whitespace review; run relevant
example tests if executable examples changed. Reuse Stage 28's successful
code/race validation when its covered content is unchanged.
## Stage 30 — Audit Scaling And Verify The Completed Target
Scope: a bounded implementation review and final validation, not another broad
refactor or expansion of the feature/performance scope.
1. Inspect the completed code for duplicate combination/identity algorithms,
per-record scans of whole collections, repeated declaration gathering, numeric
magnitude expansion, stale ownership/path caches, and unnecessary copies.
Prefer shared indexed mechanisms without pooling/global caches. Reuse existing
composition/underlay/inheritance benchmarks and add a focused mixed-upsert or
wildcard-collection benchmark only where existing fixtures cannot exercise the
new scaling risk. Record commands and qualitative findings here; timing or
allocation thresholds and a quiet-host benchmark campaign are not gates.
2. Confirm transitional adapters/dead early-CLI paths have been removed. Search
current code and maintained examples for `SequenceCompositions` and
`SequenceComposition`; only intentional historical references may remain.
Confirm no new dependencies, public internal-type exposure, background work,
release machinery, or additional limits were introduced. Review the full diff
against the roadmap, not merely passing tests. Record bounded review findings
here; significant corrective
work requires its own focused unit rather than being hidden inside this stage.
Do not declare completion while a target behavior remains incorrect or missing.
Validation: `./scripts/check.sh`, `go test -race ./...`, and the development
guide's local documentation/public-boundary checks, reusing successful checks
from Stages 28–29 if their covered content is unchanged. Record completed scope,
validation, benchmark commands/qualitative findings, and any real remaining
issues accurately. No new timing thresholds or release gates are introduced.
Acceptance: all roadmap operations and lifecycle boundaries work through ordinary
and layered loads; values and inspection agree; application constraints hold on
successful final results; deferred behavior and YAML-only guarantees have not
expanded. All thirty stages are complete and their nonempty units committed.
## Open Questions
None. Feature choices are settled in the roadmap and accepted ADRs; this plan
selects the remaining API, representation, validation, and execution decisions.

View File

@@ -1,999 +0,0 @@
# Sequence Identity And Combination Roadmap
Status: settled feature roadmap; implementation is specified in the
[staged implementation plan](implementation.md). This document defines the
targeted end state, not an implemented contract. The implementation plan owns
execution order, exact Go names and signatures, bookkeeping, and validation.
## Purpose And Scope
Allow configuration authors to assemble collections from independently authored
records, declare identity in configuration, and progressively extend a baseline
collection through higher layers. Authors can replace the entire collection,
append distinct records, or replace complete records by identity while retaining
unmatched records. Sibling inheritance uses the same sequence operations and
identity-based defaults. Within an identified sequence, a record can explicitly
inherit settings from another record in that sequence while keeping its own
identity.
The motivating use case is a prompt catalog whose source-backed baseline supplies
records and whose application or installation layers add or supersede records.
Applications can optionally impose identity constraints that configuration must
respect and that hold in the final loaded configuration. These constraints can
cover exact destinations or collections beneath dynamically named mapping keys,
without requiring the application to discover those names before loading.
The feature covers ordinary and layered loading, imported items, inline
sequences, selected overlays, and sibling inheritance. Typed defaults, CLI
assignments, and application resolution retain their own combination semantics;
CLI application moves later in the loading lifecycle.
This scope is narrower than the deferred
[identified-record sequences proposal](future.md#1-identified-record-sequences).
Automatic recursive merging of matching records remains outside this scope;
explicit inheritance between distinct records is included. The initial
declaration surface is `configkit.sequences`; `key` and `mode` shorthand on
imports is deferred. Application lists remain
ordinary YAML sequences without control wrappers. Unkeyed sequence concatenation
is outside this feature.
## Current Baseline And Required Changes
The [import contract](../config.md#explicit-imports) owns existing selectors,
destinations, placement, and ordering. `placement: sequence-item` already places
each selected source mapping as one item at its effective destination. Item-local
helper imports can contribute fields before placement.
The [composition contract](../config.md#strict-document-composition),
[layered-loading contract](../config.md#layered-loading), and
[inheritance contract](../config.md#sibling-inheritance) describe the implemented
baseline:
| Area | Current behavior | Target behavior |
| --- | --- | --- |
| Identity declaration | `Definition.SequenceCompositions` registers one literal string field at each registered mapping path. | YAML collection declarations and optional authoritative application registrations support field sets and scalar identity components. |
| Application registration destinations | Registrations select exact literal mapping paths. | Exact and structural selectors can impose identity beneath arbitrary mapping keys, independently at each matched destination. |
| Strict composition | Registered sequences append distinct string identities and reject duplicates; repeated unregistered sequences conflict. | Collections with effective identity append distinct records and reject duplicates. Applicable lower-tier identity is available before composing higher-tier peers. Unidentified sequences retain strict-conflict behavior. |
| Source precedence and selected overlays | A present higher sequence replaces the lower sequence wholesale. | Collection policy selects `replace`, `append`, or `upsert`; overlays can declare policy for inline sequences. |
| Sibling inheritance | An inheriting mapping recursively overlays its resolved parent; supplied sequences replace wholesale. | Mappings retain recursive combination, while sequences use destination policy and the same three operations. |
| Sequence-member inheritance | A sequence element has no sibling scope for its own inheritance relationship. | An identified sequence provides a sibling scope: a record can inherit from one record in the same sequence, selected by complete identity. |
| CLI overrides | Apply before inheritance and typed defaults/decoding. | Apply after inheritance and typed defaults/decoding, before application resolution. |
| Identity enforcement | Registration governs strict sequence composition. | Structural identity checks cover identified combinations; application registrations additionally constrain final effective sequences. |
Source-backed underlays participate in source assembly. `Definition.Defaults`
provides typed defaults later; extending source combination does not turn typed
default lists into source underlays.
## Collection Declarations
Use a source-root `configkit.sequences` block containing collection declarations.
Each entry identifies a `destination` and may declare `key`, `mode`, or both.
Identity and mode can be supplied independently. This form serves imported items,
inline records, ordinary mapping imports, and inline sequences in selected
overlays.
```yaml
configkit:
sequences:
- destination: [prompts]
key: [namespace, id]
mode: append
imports:
- dir: prompts
pattern: '*.yaml'
destination: [prompts]
placement: sequence-item
```
Each selected file contributes a record such as:
```yaml
namespace: support
id: welcome
text: Welcome
```
Here append retains a lower-layer collection and adds distinct higher-layer
records. Identity and combination controls belong to the collection declaration,
so imported and inline contributions share one policy surface.
### Destinations And Scope
`destination` uses the existing sequence of literal mapping keys. Resolve it
relative to the declaring source's placement: a fragment mounted under `catalog`
can declare `[prompts]` locally. A sequence-item source can declare a nested
collection within its own item without referring to an eventual numeric index.
Preserve source and item-local scope boundaries.
A collection's scope is its resolved destination within its composition scope.
Resolve relative destinations before reconciling declarations. Different imported
items retain separate scopes for their nested collections. A declaration governs
all contributions to its collection in the tier, including imported and inline
records; it does not govern only records in the declaring file.
Gather declarations before composing the relevant scope so their effect does not
depend on which contribution is assembled first. Retain applicable controls
through source precedence and inheritance, then keep them out of application
decoding.
Sequence declarations are independent of filesystem import enablement. Selected
overlays may contain `configkit.sequences`, extending the current blanket
prohibition on overlay `configkit` declarations. This does not enable imports
in overlays.
### Overlay Authoring
For an existing sequence `[foo, bar]`, an overlay can replace `foo` in full:
```yaml
configkit:
sequences:
- destination: [prompts]
key: id
mode: upsert
prompts:
- id: foo
text: Revised definition
```
The result is `[foo*, bar]`. The explicit upsert mode can be omitted when no
different explicit mode applies from a lower tier: effective identity then
selects upsert. To add a distinct record instead:
```yaml
configkit:
sequences:
- destination: [prompts]
key: id
mode: append
prompts:
- id: baz
text: Another prompt
```
The result is `[foo, bar, baz]`. To replace the whole sequence:
```yaml
configkit:
sequences:
- destination: [prompts]
mode: replace
prompts:
- id: bar
text: Bar definition
- id: baz
text: Baz definition
```
The result is `[bar, baz]`. Explicitly selecting the default operation is valid.
With no applicable explicit mode, omitting this declaration selects replace for
an unidentified collection and upsert for an identified one. A retained explicit
mode continues to govern.
## Identity Definitions And Constraints
### Field Sets
`key` accepts a string or a nonempty list of strings naming direct, literal
fields in each record. A name such as `metadata.id` denotes that literal field,
not a nested lookup. Treat field names as a set: remove repeated names and
ignore authored order when comparing definitions.
Thus `id`, `[id]`, and `[id, id]` are equivalent; `[namespace, id]`,
`[id, namespace]`, and `[namespace, id, id]` are also equivalent. An empty list
is invalid because it supplies no identity fields. These rules apply to both
configuration declarations and application registrations.
Use a consistent field order internally to construct identity tuples. Preserve
component types and tuple boundaries so unstructured concatenation cannot cause
otherwise distinct identities to collide.
### Scalar Components And Equality
Each identified record must be a mapping with every designated identity field
present. Components may be strings, numbers, booleans, or explicit null. Missing
fields and mapping or sequence components are invalid. Present null differs
from absence; two single-field null identities are equal. Compound identities
are equal when all corresponding components are equal.
Strings compare exactly, including empty strings, without trimming, case folding,
or normalization. Scalar categories remain distinct: `"1"`, `1`, `true`, and
null are different identities.
Numbers compare by mathematical value across integer, fractional, and exponent
spellings: `1`, `1.0`, and `1e0` are equal, as are `0` and `-0`. Use exact numeric
normalization for identity comparison without rounding through a floating-point
conversion. This does not rewrite authored values or change existing typed
decoding and projection semantics. The parser's rejection of non-finite numeric
forms remains unchanged.
### Application-Owned Constraints
Retain an optional application interface for registering collections and imposing
their identity definitions. Configuration may define identity where no application
constraint exists. When a declaration omits `key`, an applicable registration
supplies identity. An explicit equivalent definition is accepted, and a different
definition fails under every mode, including replace.
For example, if the application requires `id`, `key: [id, name]` is invalid:
that compound identity would allow repeated `id` values with different names.
A registration is an authoritative constraint, not an omission-filling default.
Copying a collection through inheritance does not create another application
registration; registrations constrain their registered destinations.
Identity constraints validate actual sequences. They do not independently require
a path to exist, be non-null, or have sequence shape. The application schema and
validation own those requirements. Final-result enforcement for application
registrations is defined under [Validation And Diagnostics](#validation-and-diagnostics).
### Application Registration API
Use one public sequence-identity registration type, collected in one field on
`Definition`. Each registration contains an explicit destination selector and
its identity fields. Exact and dynamic registrations share matching, overlap
validation, identity comparison, and lifecycle behavior.
Provide two constructors returning that same registration type:
- An exact-path convenience constructor accepting a literal `Path` and identity
field names. It converts the path into a selector containing literal segments.
- An explicit-selector constructor accepting a structural selector and identity
field names, for destinations that include arbitrary mapping keys.
Both constructors accept identity fields as variadic strings, so one-field and
compound identities use the same construction form. Apply the settled field-set
rules, including deduplication. Invalid registrations, including an empty identity
field set, fail during application-definition validation.
Existing `Path` values retain their literal meaning throughout ConfigKit. The
exact-path constructor supplies convenience at the registration boundary; dynamic
matching uses the explicit selector type. Both forms can appear together in the
same definition field.
Replace the current registration shape with this model. Preserving the existing
`SequenceComposition` fields and `Definition.SequenceCompositions` spelling is not
a requirement, and no compatibility facade is needed solely to retain that syntax.
Choose names centered on sequence identity, reflecting its role in composition,
inheritance, and final validation. Exact Go type, field, and constructor names
and signatures belong in the implementation plan.
### Dynamic Application Registration Destinations
Allow applications to register identity for collections whose containing mapping
keys become known only during loading. For example:
```yaml
collections:
weatherreporter:
prompts:
- id: foo.bar
text: Report the weather.
homewatch:
prompts:
- id: foo.bar
text: Report the household status.
```
One application registration can require `id` for every namespace's `prompts`
list. The equal IDs above are valid because each list has its own uniqueness
scope. An optional root `prompts` list can have a separate exact registration.
Use an explicit application-side structural selector composed of literal mapping
keys and an arbitrary-mapping-key segment. Conceptually, the namespace rule is:
```text
Literal("collections") -> AnyMappingKey -> Literal("prompts")
```
This notation explains the selector; it is not a Go signature or YAML syntax.
Each segment consumes exactly one mapping level. Match the complete destination,
not an arbitrary prefix or suffix. Literal keys remain exact, including empty
strings and keys containing punctuation such as `*`, `/`, or `.`. An arbitrary-key
segment is distinct from any literal spelling and does not match a sequence
index. Existing `Path` values retain their literal meaning.
This capability belongs to the reusable application definition and applies to
ordinary and layered loads. YAML collection destinations and import destinations
retain their existing literal syntax. Recursive search, regular expressions,
filesystem glob rules, sequence-index selectors, and custom matching callbacks
are outside this addition.
At each concrete matched destination, apply the same authoritative identity
constraint as an exact registration. Use the settled field-set and scalar-equality
rules. Do not pool identities across destinations or require records to repeat
the containing mapping key as an identity component. Applications retain ownership
of namespace interpretation, qualified runtime IDs, and domain validation.
Matching supplies effective identity before relevant strict composition and
precedence operations. Without an applicable explicit mode, this identity selects
upsert under the existing default rule. It also supplies identity for inheritance
within the matched sequence. YAML declarations must agree with it under every
mode, including replace. These are ordinary application constraints with a broader
destination selector, not a separate final-validation facility.
### Registration Overlap
Equivalent application constraints coalesce at any shared destination. Equivalence
uses the identity field-set rules: for example, `id` and `[id, id]` agree.
Incompatible constraints fail; exact registrations do not override broader
selectors, and registration order grants no precedence.
Detect incompatible overlapping registrations while validating the application
definition, before source acquisition. For these fixed-depth mapping selectors,
overlap exists when the selectors have the same number of segments and every
corresponding pair can match a common key. Unequal literal keys at any position
make the selectors disjoint; an arbitrary-key segment can overlap a literal or
another arbitrary-key segment. Compare segment structure rather than rendered
path strings. No configuration pre-read is needed.
Thus an `id` rule for all namespace prompt lists conflicts with a `[id, name]`
rule for one named namespace, even if that namespace is absent from a particular
load. This is an invalid application definition. A valid selector matching no
collections is harmless: registration does not require a path to exist, create
collections, or impose shape and nullability requirements.
### Matching Across The Loading Lifecycle
Match application selectors against effective configuration destinations after
accounting for input placement. Physical filenames and source-local paths do not
determine applicability. Make constraints available before composing contributions
at those destinations, including namespaces first introduced by imports, mounted
inputs, or higher tiers. Apply matching constraints to key-only YAML declarations
as well, even before their sequence exists. Consumers need neither a preliminary
load nor source inspection to enumerate namespaces.
During inheritance, apply registrations at the inheriting destination, including
newly created matching destinations. Copying a collection does not transfer its
application registrations to a different path. The existing YAML-policy ownership
and transfer rules remain separate from application selector matching.
During final validation, find matching actual sequences in the final effective
configuration. This includes namespaces introduced or replaced by typed defaults,
CLI overrides, and application resolution. Reusing only destinations found during
source assembly would miss these values. Use the same selector meaning and
identity validator in both phases; later assignments do not trigger composition
or inheritance again. YAML-only identities retain their existing enforcement
boundary.
## Strict Composition And Precedence Operations
### Compose Peers Before Applying Precedence
Resolve applicable identity and mode policy, strictly compose the tier's peer
contributions, then apply one selected operation per collection against the
accumulated lower value. The tier operation does not sequentially replace peer
source contributions. Selected overlay contributions use the same combination
semantics.
Within a strict tier, equivalent identity definitions coalesce and different
field sets fail. Agreeing explicit modes coalesce; conflicting explicit modes
fail. Omission expresses no preference. A root declaration or a later declaration
has no precedence over conflicting peers. Explicitly selecting a default mode
is still a real instruction and can conflict with another explicit mode.
Identified peer sequences append distinct records. Duplicate identities among
peers or within a contributed list fail under all three modes, including replace
and upsert. Equal or disjoint record payloads do not excuse a duplicate identity.
Unidentified sequences retain their current strict-composition conflict behavior.
For example, higher-tier fragments `[C]` and `[D]` with an agreed identity assemble
as `[C, D]`. Against lower `[A, B]`, append or upsert produces `[A, B, C, D]`,
while replace produces `[C, D]`. Peers `[B]` and `[B*]` conflict even under
upsert because neither has higher precedence.
### Available Operations
Use one spelling for each mode: `replace`, `append`, and `upsert`.
| Mode | Behavior at source-precedence and inheritance boundaries |
| --- | --- |
| `replace` | Replace the complete lower sequence with the assembled higher sequence. |
| `append` | Retain lower records and append higher records; duplicate identities fail. |
| `upsert` | Replace matching records in full, retain unmatched lower records, and append new identities. |
Append and upsert require effective identity. ConfigKit does not infer identity
from an `id` field or record position. Whole-sequence replacement does not require
identity agreement with lower YAML policy, but application constraints still apply.
Letters denote record identities; `B*` has the same identity as `B` with a different
payload:
| Lower | Higher | Mode | Result |
| --- | --- | --- | --- |
| `[A, B]` | `[C]` | `append` | `[A, B, C]` |
| `[A, B]` | `[C]` | `replace` | `[C]` |
| `[A, B]` | `[B*]` | `append` | Duplicate-identity error |
| `[A, B]` | `[B*]` | `replace` | `[B*]` |
| `[A, B]` | `[B*]` | `upsert` | `[A, B*]` |
| `[A, B]` | `[B*, C]` | `upsert` | `[A, B*, C]` |
Upsert replaces the whole matched record. If lower `B` has `id`, `text`, and
`temperature`, while higher `B` has only `id` and `text`, the result at this
boundary has only `id` and `text`. Subsequent defaulting or other lifecycle
operations retain their own semantics. Mappings continue to combine recursively;
matching sequence records do not recursively merge fields.
### Mode Selection And Persistence
Select the operation in this order:
1. Reconcile explicit modes within the current tier; conflicting peers fail.
2. Use the current tier's explicit mode if supplied.
3. Otherwise retain an applicable explicit mode from a lower tier.
4. With no explicit mode, infer upsert when effective identity is available.
5. Otherwise infer replace.
Effective identity can come from an application registration, current declarations,
or applicable lower policy, subject to the identity agreement and lifetime rules
below. Determine the inferred mode from available identity before applying the
operation. A plain higher-tier sequence can therefore use upsert without repeating
a key declaration.
An explicit mode persists at its applicable destination until a higher-priority
explicit declaration changes it. New sequence values alone do not reset it, and
an unrelated overlay leaves it intact. A mode-only declaration can change policy
without supplying records; it neither clears the collection nor retroactively
recombines earlier tiers. Later source contributions and inheritance use the
resulting policy.
| Lower explicit mode | Higher declaration | Effective mode |
| --- | --- | --- |
| `append` | Mode omitted | `append` |
| `append` | `upsert` | `upsert` |
| `upsert` | `replace` | `replace` |
| None | Mode omitted, effective identity available | Inferred `upsert` |
| None | Mode omitted, no effective identity | Inferred `replace` |
Only explicit modes persist. Inferred defaults are recomputed from effective
identity when needed. Explicit replace continues to apply in subsequent tiers
until another mode is declared; introducing identity alone does not override it.
Mode persistence does not preserve YAML identity discarded by replacement, nor
policies owned by discarded records. Source precedence and inheritance use the
same destination policy, without separate per-phase settings. CLI assignments
remain outside mode selection.
### Presence, Empty Sequences, Null, And Kind Mismatches
At a collection's precedence boundary:
| Higher contribution | `replace` | `append` / `upsert` |
| --- | --- | --- |
| Absent | Leave the lower value unchanged. | Leave the lower value unchanged. |
| Explicit `[]` | Clear the sequence. | Contribute no records. |
| Explicit null or another non-sequence value | Ordinary replacement, subject to application schema. | Error: the operation requires a sequence. |
A declaration alone does not clear a collection. An optional import selecting
nothing contributes nothing and does not erase existing records. An absent lower
collection can be initialized by an incoming sequence under any mode, subject to
identity requirements. Explicit lower null or another non-sequence value is not
absence: append and upsert fail against it, while replacement remains available.
An empty incoming sequence bypasses neither kind checks nor identity requirements.
With effective identity and no explicit mode, `prompts: []` uses upsert and does
not clear the collection. Clearing requires replace, selected in the current tier
or retained from lower policy. Replacing an identified collection with null also
requires replace and a schema permitting null. Collection null is distinct from
a present null identity component inside a record.
### Ordering
Retain existing dependency-ready source scheduling within each tier and authored
item order within lists. Gathering declarations does not change assembly order;
identity values do not sort records.
- Append keeps lower records in order, followed by higher records in assembled order.
- Replace keeps only higher records in their assembled order.
- Upsert keeps a matched record's lower position and appends new identities in
higher-tier order.
For example, lower `[A, B, C]` and higher `[C*, A*, D]` produce `[A*, B, C*, D]`
under upsert. Whole-sequence replacement gives the higher tier control of the
complete sequence order.
## Policy Agreement, Lifetime, And Ownership
### Identity Agreement And Replacement
Append and upsert use one agreed identity for all participating records:
| Lower identity | Higher declaration | Result when combining records |
| --- | --- | --- |
| `id` | Omitted | Use `id`. |
| `id` | `id`, `[id]`, or `[id, id]` | Accept the equivalent definition. |
| `id` | `[namespace, id]` | Error: different definitions. |
| None | `id` | Establish `id` and validate all participating records. |
Introducing identity for an unkeyed lower sequence requires checking both lower
and incoming records, including uniqueness within each contribution. With no
identity from the application or either participating collection, append and
upsert fail.
YAML identity follows the collection that survives an operation:
| Operation | Effective YAML identity |
| --- | --- |
| Append or upsert | Retain or establish the agreed identity. |
| Whole-sequence replacement | Discard lower YAML identity and use the replacing tier's identity declaration, if any. |
| Replacement by null or another non-sequence value | Discard lower YAML identity and retain any identity explicitly declared by the replacing tier. |
Application constraints remain authoritative under every operation. Without an
application constraint, replacement can introduce a different identity, such as
changing from `id` to `name`. Replacing a sequence without a new key declaration
or application registration produces an unkeyed sequence. No identity-removal
directive is needed.
The reset occurs only when the collection value is actually replaced. Declaring
replace without supplying a replacement does not authorize changing the identity
of retained records. Explicit mode lifetime is separate from YAML identity
lifetime.
### Ancestor Replacement
Replacing an enclosing mapping can remove a collection without replacing that
collection directly. Apply the same identity and ownership rules to this removal:
discard the removed collection's lower YAML identity, and discard policies owned
by any removed records. Reintroducing the same path later does not restore those
discarded policies.
Independently declared explicit modes retain their existing persistence rules.
Removing a descendant collection does not remove a mode whose declaring scope
survives. Application registrations remain applicable at matching destinations,
including destinations reintroduced later. Identity newly declared by the
replacing tier follows the existing declaration and ownership rules.
For example, a root-level declaration establishes `key: id` and `mode: append`
for `[catalog, prompts]`. A higher tier replaces `catalog` with null, removing
the prompts collection and its YAML identity. The independently declared append
mode survives. If a later tier recreates `catalog.prompts`, append requires
identity supplied by a new declaration or an application registration. Without
that identity, loading fails; the later tier can instead explicitly choose
replace. The replacement of `catalog` itself follows ordinary ancestor-value
precedence and does not attempt to append null to the removed prompts list.
### Declarations Without Sequence Values
A key declaration without accompanying records applies immediately to an existing
retained sequence: check agreement and validate its records, without recombining
earlier tiers. For example, a higher tier that only declares
`destination: [prompts], key: id` identifies and validates the retained lower
`prompts` sequence.
If no collection exists, retain the declaration in its owning scope for later
contributions. An otherwise empty underlay can establish identity for main-tier
fragments. An explicit identity declaration can also remain pending at a
non-sequence destination; it does not independently impose a shape requirement.
The presence and kind rules still determine whether an attempted operation fails.
Item-local declarations retain their ownership boundaries.
### Lower-Tier Identity During Strict Composition
Resolve applicable identity before composing a higher tier's peers, using
application registrations, available lower-tier policy, and declarations gathered
within the higher tier's scope. Authors need not repeat a baseline key solely
because the higher tier has multiple fragments.
For example, an underlay establishes `key: id` for `prompts`, and two main-tier
fragments contribute `[foo]` and `[bar]` without a key declaration. They strictly
compose as `[foo, bar]` before the main tier combines with the accumulated lower
collection. If both contribute `foo`, composition fails; upsert does not give
precedence to peers.
Lower YAML identity remains subject to agreement, replacement, and ownership.
A replacing collection must not be constrained by obsolete lower identity just
because policy resolution precedes composition. Likewise, a replacement record
cannot borrow nested identity owned by the discarded lower record, even when
outer identity or numeric position matches. Application constraints still apply.
Higher-tier declarations do not retroactively repair invalid lower-tier composition.
### Nested Record Ownership
Attach item-local policies to their owning contributions so they follow surviving
records independently of numeric indexes. Whole-record upsert discards the old
record's local policies along with its fields. Whole-sequence replacement discards
policies owned by removed records. A replacement record supplies its own local
declarations; equal outer identity or final index does not transfer discarded
policies.
For example, a source imported as one record into `prompts` can declare:
```yaml
configkit:
sequences:
- destination: [steps]
key: id
mode: append
id: foo
steps:
- id: prepare
```
The declaration governs `steps` inside that `foo` record. Retaining `foo` retains
its local policy, including the explicit mode. Upserting a new `foo` discards the
old `steps` and its declaration even though the outer record keeps its position.
Policies independently applicable at a destination are not discarded merely
because an individual record is replaced. Ownership determines which declarations
are removed. Application registrations remain constraints at their registered
destinations. Inheritance copies transfer identity and choose operations as
specified next.
## Sibling Inheritance
Complete source-tier assembly before resolving inheritance. Retain existing
mapping-sibling scopes and add identified-sequence scopes as described below.
Both use the existing dependency-ready algorithm. At each inheritance combination,
the resolved parent's collection is the lower value and the inheriting mapping's
local collection is the higher value. Mappings combine recursively; upsert within
an identified sequence remains whole-record replacement. Explicit record
inheritance is a separate construction step after those replacements.
Use the inheriting destination collection's policy: an applicable explicit mode,
including one retained through source tiers, wins. Otherwise infer upsert with
effective identity and replace without it. There is no separate default-append
rule for inheritance. The same destination policy governs source-precedence and
inheritance combinations.
YAML identity follows a collection copied through inheritance, subject to agreement
and replacement rules. A source collection's mode does not automatically become an
instruction at a different inheriting destination. For example, source `steps`
has `key: id` and `mode: append`; an inheriting mapping supplies local `steps`
without an applicable explicit mode. The copied identity selects upsert there.
An explicit destination append instead selects append. A matching destination key
agrees; a different key conflicts when combining records, while explicit replace
can establish new YAML identity subject to application constraints.
For example:
```yaml
configkit:
sequences:
- destination: [parent, items]
key: id
child:
items:
- id: foo
value: A
- id: bar
value: B
parent:
extends: child
items:
- id: baz
value: C
```
Despite the names, `parent` inherits from `child`. Identity at `[parent, items]`
selects upsert, producing `[foo, bar, baz]`. If the local sequence instead supplies
a revised `foo`, the result is `[foo*, bar]`. Append rejects that repeated identity;
replace retains only the local sequence.
The same ordering, presence, and kind rules apply at inheritance boundaries.
An omitted local sequence retains the inherited sequence. Explicit `[]` clears it
under replace and contributes no records under append or upsert. Retain controls
until inheritance uses them, then exclude them from application decoding.
### Inheritance Between Identified Sequence Records
An identified sequence supplies a sibling scope for its records. Identity can
come from an application registration or an applicable YAML declaration. A record
may inherit from one other record in that same sequence using an `extends`
mapping that selects the parent's complete identity:
```yaml
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: base
model: standard
temperature: 0.5
- id: concise
extends: {id: base}
temperature: 0.2
```
The final `concise` record is:
```yaml
id: concise
model: standard
temperature: 0.2
```
The `base` record remains in the collection. Parents are ordinary records subject
to the same decoding and validation as other records; there is no abstract
template category.
Use mapping-form references for both single and compound keys. With
`key: [namespace, id]`, a reference can be
`extends: {namespace: shared, id: base}`. Match components using the collection's
field-set and scalar-equality rules, including mathematical numeric equality.
`extends: {id: null}` selects a null-valued identity; `extends: null` cancels a
relationship.
Parent-reference mappings use ordinary strict mapping composition. Item-local
helper imports may supply disjoint fields of a compound reference; overlapping
fields conflict, even when equal. For example, a helper's
`extends: {namespace: shared}` and the importing record's `extends: {id: base}`
compose into `extends: {namespace: shared, id: base}`. This assembles one record;
it does not permit strict peers to contribute duplicate record identities.
Before parent lookup, require the surviving effective reference mapping to contain
exactly the collection's identity fields, with supported scalar values. Missing
or extra fields fail. Validate the completed reference during inheritance, rather
than requiring each helper contribution to be independently complete. Superseded
references remain historical evidence; earlier strict-composition conflicts
remain errors.
Every record must supply its complete own identity before inheritance. Preserve
the child's identity fields and inherit only the parent's nonidentity payload;
inheritance cannot supply a missing child identity. Child payload fields take
precedence, mappings combine recursively, and nested sequences use the inheriting
destination's collection policy. Consume the record's directive before decoding.
Support one parent within the same identified sequence, including forward
references and chains. Missing parents and cycles, including self-reference,
fail the load. Cross-collection references, position-based references, multiple
parents, and inheritance between unidentified sequence members are outside this
scope. Existing string-form references between mapping siblings retain their
meaning; they do not become shorthand for sequence identity references.
Resolve relationships against records surviving source-tier assembly and selected
overlays. A higher layer can replace a parent before inheritance runs; removing
that parent makes a surviving reference fail. Whole-record upsert discards the
old record's directive along with its payload. Omitting `extends` in the
replacement record does not retain the displaced record's relationship. An
explicit reference to the record's own identity is a cycle, not a way to access
its lower-tier predecessor.
Reuse Kahn's algorithm with the existing stable canonical semantic-pointer key
for dependency-ready ordering. Build parent lookup from the sequence's effective
identity, and preserve the assembled list order independently of resolution
order. Enclosing relationships resolve before nested relationships. Parent
templates retain nested instructions for evaluation at each inheriting
destination, following the existing destination-local inheritance rule. Copied
collection identity, destination operation selection, and record-owned policy
transfer follow the rules above.
Record inheritance runs in the existing structural phase, before typed defaults,
decoding, CLI overrides, and application resolution. Those later stages do not
trigger another inheritance pass. Preserve inherited presence, origin and
history evidence, sensitivity, declaring resource contexts, and existing
expansion budgets just as for mapping-sibling inheritance. Identity lookup must
not expose identity values in diagnostics. Existing index-based inspection paths
remain sufficient; no new identity-selector path API is needed.
## Loading Lifecycle
Use the following order:
1. Assemble source tiers and selected overlays, resolving policy before each
tier's strict composition and applying collection operations at precedence
boundaries.
2. Resolve sibling inheritance in mappings and identified sequences, then apply
typed defaults and decode the assembled configuration.
3. Apply explicit CLI overrides to that configuration and check supplied values
against the application schema.
4. Run application resolution against the overridden configuration.
5. Resolve effective secret references, validate the final configuration including
application-registered identities, and publish it.
### Typed Defaults And Application Resolution
Source-backed underlays, including application-supplied mapping sources,
participate in sequence combination. Typed `Definition.Defaults` remains the
initial decoded state: supplied sequences replace default sequences. Source
append does not append to a typed default list.
Application resolution retains assignment and recorded-transformation semantics.
ConfigKit does not automatically combine a callback's assigned sequence with its
previous value. Application-registered identities receive final validation
independently of how defaults, CLI overrides, or callbacks produce their values.
### CLI Precedence
CLI overrides are the highest-precedence input to application resolution.
Supplied sequences replace whole sequences, including sequences inside supplied
mappings. Mappings retain recursive override behavior; empty sequences and
permitted nulls retain explicit replacement meaning. Configuration append or
upsert does not reinterpret CLI assignments. Inheritance and defaulting do not
run again afterward: overriding assembled `[A, B]` with `[C]` presents `[C]` to
application resolution without inheritance reintroducing prior records.
Application resolution sees CLI values and their explicit-presence and source
metadata. `Resolution.Default` preserves explicit CLI values; normalization and
recorded application transformations may still change them. For example, a derived
URL can use a CLI hostname, while an explicit CLI URL can be preserved by an
absence-based default. Retain recording and provenance requirements for those
transformations.
Reject structural directives such as `extends` and `configkit` in CLI override
paths or supplied values. CLI input supplies application values after structural
processing; it cannot supply or cancel inheritance directives, change completed
relationships, or prevent earlier inheritance failures.
Typed decoding also precedes CLI application, so a CLI replacement cannot repair
an earlier decoding failure. A source string in an integer field fails before a
valid CLI integer can apply. This changes current override-before-decoding
behavior. Semantic validation remains after overrides and application resolution.
Secret lookup uses effective references after overrides and resolution. Neither
stage bypasses final application identity constraints or semantic validation.
## Validation And Diagnostics
### Structural Identity Checks
Allow item-local helper imports to complete and consume applicable non-main
source-version headers before checking payload identity for collection assembly.
A helper can supply an identity component; a consumed header cannot supply one
after removal.
Structural controls are likewise unavailable as source identity components.
Check identity against application payload fields, excluding fields acting as
structural directives even when the engine retains them for later processing.
For example, `key: extends` with an item containing `extends: null` does not
supply a null identity: that field cancels inheritance and is consumed. Report
the ordinary missing-identity-component error at the collection/item path using
available origins, without exposing values. Do not defer this failure until
directive consumption or final application validation.
This rule follows a field's structural role; it does not introduce a broader
restriction on unrelated literal field names. Final application identity checks
continue to examine the effective typed payload under their existing rules.
Validate each identified contribution before combination and check uniqueness in
the resulting identified collection, including inheritance operations. Upsert
matches across precedence are replacements; duplicates within a contribution
remain errors. Later inheritance, typed defaults, or callbacks cannot repair an
identity missing at an earlier assembly boundary.
YAML-only identities are enforced during structural composition, precedence
assembly, and identified inheritance operations. They have no final-result
guarantee after typed defaulting, CLI overrides, or application resolution. Those
stages can produce duplicate identities or otherwise violate a YAML-only identity.
Applications needing a final guarantee register the constraint through the
application-owned interface.
Adding record inheritance does not expand this guarantee: a YAML declaration
can enable parent lookup without imposing final validation on values later
produced by defaults, CLI overrides, or application resolution.
### Final Application Constraints
On successful loading, final effective collections covered by application
registrations satisfy their declared identity constraints. Retain assembly-time
checks and validate final effective sequences before publication, after defaults,
inheritance, overrides, application resolution, and secret lookup.
This covers both exact and dynamic application registrations. Evaluate dynamic
selectors against the final effective configuration, with independent validation
for each concrete matched sequence as described under
[Matching Across The Loading Lifecycle](#matching-across-the-loading-lifecycle).
ConfigKit performs these identity checks; consumers register the constraint
without needing to repeat duplicate detection in `Definition.Validate`.
Application-owned semantic validation continues to cover domain-specific rules.
This guarantee covers actual sequences, not path existence or allowed shape.
It does not require every intermediate callback state to satisfy the constraint
and does not cover caller mutations after loading returns. Final checks examine
effective typed values; assembly checks validate authored contributions. A final
scalar alone does not prove that a source explicitly supplied that field.
Use the existing canonical final-value representation and shared publication
path as the foundation. The guarantee requires neither a new mutation-history
system nor exact causal reconstruction for records supplied by defaults or
callbacks.
### Diagnostics And Inspection
Identity diagnostics identify collection and item paths using available origin
information without exposing identity values. Conflicting declarations identify
the collection destination and declaration locations.
For dynamic registrations, use concrete matched collection and item paths for
value diagnostics, with available origins and existing sensitivity rules.
Definition-time overlap errors identify the conflicting registrations; they do
not require or invent a concrete destination from a loaded document.
Correctness includes final values, projected item paths, source destinations,
provenance, history, sensitivity, and passive resource contexts. Appending can
change final indexes; replacing changes which contributions survive. Preserve
the existing loading and inspection contracts through both operations, including
policy ownership and inheritance transfer.
## Work Needed From The Baseline
The baseline needs the following connected capabilities to reach the target
state. Execution order and test sequencing belong to the
[implementation plan](implementation.md):
- **Declarations and public API:** extend application registration to the settled
identity model with one public registration type and definition field, an
exact-path convenience constructor, and an explicit-selector constructor.
Preserve literal `Path` semantics. Parse and consume `configkit.sequences` across
supported source forms and selected overlays; preserve placement-relative and
item-local scopes. Keep sequence declarations independent of filesystem import
enablement.
- **Identity and policy representation:** implement field-set comparison, typed
scalar identity tuples with exact numeric equality, declaration agreement,
pending declarations, explicit-mode persistence, replacement lifetime, and
record-owned policy transfer. Choose bookkeeping proportional to these semantics.
Handle descendant collection removal through ancestor replacement, preserving
independently declared modes and application constraints while discarding
removed collection identity and record-owned policies.
- **Registration matching:** support literal and arbitrary-mapping-key segments,
detect incompatible selector overlap during definition validation, and resolve
applicable constraints at concrete placed destinations. Reuse the identity
rules and validator for exact and dynamic registrations. Keep matching separate
from source discovery and from YAML declaration ownership.
- **Assembly handoffs:** make accumulated lower policy available before higher-tier
strict composition. Separate source acquisition from policy-aware composition
or otherwise revise assembly sequencing; current main-before-underlay composition
is not a design constraint. Preserve distinctions among acquisition order,
dependency-ready scheduling within tiers, and precedence order. Account for
source records, callbacks, main-version checks, and failure diagnostics. Retain
version ownership and validation semantics; higher replacement cannot hide
lower-tier structural failures.
- **Combination and inheritance:** implement the three operations with strict peer
handling, whole-record replacement, presence rules, stable ordering, and the
destination policy rules for inheritance. Extend sibling-scope discovery and
parent lookup to identified sequences, validate mapping-form references, and
preserve child identities while copying parent payloads. Reuse the shared
dependency-ready scheduler and destination-local nested resolution. Preserve
provenance, history, sensitivity, paths, and resource contexts as records survive,
move, or disappear.
- **Lifecycle and validation:** move CLI application after inheritance and typed
defaults/decoding, preserve explicit-presence and source metadata for application
resolution, reject CLI structural directives, and add final application identity
validation through the shared publication path. Keep YAML-only enforcement
within structural operations. Match dynamic registrations during assembly and
inheritance, and against the final effective configuration rather than a saved
list of source-backed destinations.
Exclude structural controls from source identity checks even while retaining
those controls for later inheritance processing.
- **Documentation and verification:** exercise ordinary and layered loading,
imported and inline collections, item-local scopes, version-header consumption,
selected overlays, inheritance, typed defaults, CLI, callbacks, and inspection.
Cover single and compound parent references, scalar equality, forward references
and chains, missing parents and cycles, parent replacement/removal, child identity
preservation, nested destination policy, cancellation, and stable output order.
Verify helper-composed references, overlapping reference fields, missing or extra
fields, and replacement of references together with their containing records.
Verify independent namespace uniqueness, root and namespaced registrations,
literal punctuation keys, equivalent and incompatible overlaps, and conflicts
detected even without a matching input destination. Cover namespaces introduced
by imports, placement, higher tiers, inheritance, defaults, CLI, and callbacks;
preserve ordinary strict-peer and upsert behavior. Check YAML disagreement under
replace, harmless absent or empty collections, schema-owned shape requirements,
unrelated sequences remaining unselected, and concrete-path diagnostics.
Verify both constructors feed the same registration model, including mixed
exact and dynamic registrations, compound identities, and definition-time
rejection of empty identity field sets.
Cover ancestor replacement followed by collection reintroduction, including a
surviving append mode with and without renewed identity, explicit replacement,
and application-registered identity. Verify that `extends: null` cannot supply
an identity component and that structural-control exclusion uses ordinary
value-free missing-identity diagnostics.
Update the public contract and consumer examples when behavior lands.
The collection-policy and CLI-ordering decisions are accepted in
[ADR 0017](../adr/0017-collection-sequence-policy.md) for collection policy and
[ADR 0018](../adr/0018-cli-after-structural-resolution.md) for CLI ordering.
Their scoped supersession is recorded in the affected earlier ADRs, and the
[architecture policy](../policy/architecture.md) reflects the accepted decisions.
These govern the target architecture; the configuration contract continues to
describe implemented behavior until the feature lands. This roadmap does not
select migration machinery.
ADR 0017 now includes inheritance within identified sequences;
[ADR 0009](../adr/0009-sibling-inheritance.md) records that scope extension in its
status while preserving its decision content. The architecture policy is aligned.
Dynamic selectors extend the destinations covered by application registration;
they retain the accepted application authority and final-validation boundary.
No open feature decisions remain. The implementation plan translates this end
state into reviewable stages without adding guarantees, restrictions, or
combination semantics.

View File

@@ -19,6 +19,28 @@ func ExampleLoad() {
// Output: {true 3} <nil>
}
func ExampleLoad_sequenceDeclaration() {
type Item struct {
ID string `yaml:"id"`
}
type Config struct {
Items []Item `yaml:"items"`
}
result, err := configkit.Load[Config](configkit.Bytes("catalog.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
items:
- id: prompt
`)))
if err != nil {
fmt.Println(err)
return
}
fmt.Println(result.Config.Items[0].ID)
// Output: prompt
}
func ExampleNoSource() {
type Config struct {
Workers int `yaml:"workers"`
@@ -154,8 +176,15 @@ func ExampleOverride() {
})
}
result, err := configkit.Load(configkit.NoSource(), configkit.Options[Config]{
Definition: configkit.Definition[Config]{Defaults: func() Config { return Config{Workers: 4} }},
Overrides: overrides,
Definition: configkit.Definition[Config]{
Defaults: func() Config { return Config{Workers: 4} },
Resolve: func(configuration *Config, resolution *configkit.Resolution) error {
configkit.Default(resolution, configkit.Keys("workers"), &configuration.Workers, 100,
"preserve an explicitly supplied worker count")
return nil
},
},
Overrides: overrides,
})
fmt.Println(result.Config.Workers, result.Sources()[0].Label, err)
// Output: 0 --workers <nil>

View File

@@ -0,0 +1,45 @@
package consumers_test
import (
"fmt"
"os"
"path/filepath"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
type workingDirectoryPresetConfig struct {
Name string `yaml:"name"`
}
func Example_workingDirectoryPreset() {
original, err := os.Getwd()
if err != nil {
fmt.Println(err)
return
}
defer func() { _ = os.Chdir(original) }()
root, err := os.MkdirTemp("", "configkit-working-search-")
if err != nil {
fmt.Println(err)
return
}
defer os.RemoveAll(root)
if err := os.Chdir(root); err != nil {
fmt.Println(err)
return
}
if err := os.WriteFile(filepath.Join(root, "service.yml"), []byte("name: example\n"), 0o600); err != nil {
fmt.Println(err)
return
}
result, err := configkit.Load[workingDirectoryPresetConfig](
configkit.SearchConfig("service", configkit.WorkingDirectory()),
)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(result.Config.Name, filepath.Base(result.Sources()[0].Path))
// Output: example service.yml
}

View File

@@ -45,10 +45,9 @@ func LoadRequiredPromptCatalog(mainPath string) (*configkit.Result[PromptCatalog
func promptCatalogDefinition() configkit.Definition[PromptCatalogConfig] {
return configkit.Definition[PromptCatalogConfig]{
SequenceCompositions: []configkit.SequenceComposition{{
Path: configkit.Keys("prompts"),
IdentityField: "id",
}},
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
},
Validate: validatePromptCatalog,
}
}

View File

@@ -0,0 +1,145 @@
package consumers
import (
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
// SequenceCatalogConfig is a source-backed prompt catalog with an application
// supplied run label and a value derived from that label during resolution.
type SequenceCatalogConfig struct {
RunLabel string `yaml:"run_label"`
ResolvedLabel string `yaml:"resolved_label"`
Prompts []SequenceCatalogEntry `yaml:"prompts"`
}
// SequenceCatalogEntry keeps prompt meaning, resources, and credentials in the
// consuming application while ConfigKit handles collection assembly.
type SequenceCatalogEntry struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Description string `yaml:"description"`
Messages []string `yaml:"messages"`
Resource string `yaml:"resource"`
Credential configkit.Secret `yaml:"credential"`
}
// LoadSequenceCatalog searches its caller-selected main path before a custom
// fallback directory, assembles imported fragments over a file-backed underlay,
// applies one selected collection overlay, then resolves and validates once.
func LoadSequenceCatalog(mainPath, overlayName, runLabel string) (*configkit.Result[SequenceCatalogConfig], error) {
absoluteMain, err := filepath.Abs(mainPath)
if err != nil {
return nil, err
}
root := filepath.Dir(absoluteMain)
filesystem := configkit.NewFilesystem("sequence catalog", os.DirFS(root))
filesystemContext := &configkit.FilesystemContext{BaseDir: ".", Root: ".", FS: &filesystem}
mainSearch := configkit.SearchConfig("", configkit.Searches(
configkit.Paths(absoluteMain),
configkit.SearchDirectory(filepath.Join(root, "fallback"), configkit.DirectorySearchOptions{
Filenames: []configkit.FilenameGroup{configkit.CustomFilenames(filepath.Base(absoluteMain))},
}),
), configkit.SearchOptions{Required: true})
definition := configkit.Definition[SequenceCatalogConfig]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
},
Resolve: func(configuration *SequenceCatalogConfig, resolution *configkit.Resolution) error {
configuration.ResolvedLabel = "for:" + configuration.RunLabel
resolution.Record(configkit.Keys("resolved_label"), configkit.ChangeNormalization,
"derive the resolved label from the selected run label", configkit.Keys("run_label"))
return nil
},
Validate: validateSequenceCatalog,
}
return configkit.LoadLayered(configkit.Input{
Source: mainSearch,
Imports: true,
Filesystem: &configkit.FilesystemContext{Root: root},
}, configkit.LayeredOptions[SequenceCatalogConfig]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: filesystem.File("underlay.yml"),
}}}},
Overlay: overlayName,
OverlayFilesystem: filesystemContext,
Overrides: []configkit.Override{{
Path: configkit.Keys("run_label"), Value: runLabel, Flag: "--run-label",
}},
Definition: definition,
})
}
// ReadSequenceCatalogResource follows the final prompt field's source context,
// including when record inheritance retained a resource from an underlay.
func ReadSequenceCatalogResource(result *configkit.Result[SequenceCatalogConfig], promptID string) (string, error) {
if result == nil {
return "", errors.New("sequence catalog result is nil")
}
for index, prompt := range result.Config.Prompts {
if prompt.ID != promptID {
continue
}
resourcePath := configkit.Keys("prompts").Index(index).Key("resource")
info, found := result.Metadata().Lookup(resourcePath)
if !found || len(info.Origin.Locations) != 1 {
return "", errors.New("prompt resource has no single declaring source")
}
context, found := result.SourceContext(info.Origin.Locations[0].SourceID)
if !found {
return "", errors.New("prompt resource source context is unavailable")
}
data, err := fs.ReadFile(context.FS, path.Join(context.BaseDir, prompt.Resource))
if err != nil {
return "", fmt.Errorf("read prompt resource: %w", err)
}
return string(data), nil
}
return "", fmt.Errorf("prompt %q is unavailable", promptID)
}
func validateSequenceCatalog(configuration *SequenceCatalogConfig, validation *configkit.Validation) {
for index, prompt := range configuration.Prompts {
if strings.TrimSpace(prompt.Name) == "" {
validation.Error(configkit.Keys("prompts").Index(index).Key("name"), "must not be empty")
}
if len(prompt.Messages) == 0 {
validation.Error(configkit.Keys("prompts").Index(index).Key("messages"), "must contain at least one message")
}
}
if len(configuration.Prompts) == 0 {
validation.Error(configkit.Keys("prompts"), "must contain at least one prompt")
}
}
// NamespacedMapCatalogConfig demonstrates typed application maps whose
// independent prompt sequences use root and dynamic namespace registrations.
type NamespacedMapCatalogConfig struct {
Prompts []map[string]any `yaml:"prompts"`
Collections map[string]map[string]any `yaml:"collections"`
}
// LoadNamespacedMapCatalog applies the same scalar identity constraint to one
// root collection and each namespace's independently scoped collection.
func LoadNamespacedMapCatalog(source configkit.Source) (*configkit.Result[NamespacedMapCatalogConfig], error) {
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}
return configkit.Load(source, configkit.Options[NamespacedMapCatalogConfig]{
Definition: configkit.Definition[NamespacedMapCatalogConfig]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
configkit.SequenceIdentityMatching(selector, "id"),
},
},
})
}

View File

@@ -0,0 +1,239 @@
package consumers_test
import (
"bytes"
"errors"
"fmt"
"os"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/ConfigKit/examples/consumers"
)
func Example_sequenceCatalog() {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
restoreOld, restoreNew := preserveEnvironment(oldSecret), preserveEnvironment(newSecret)
defer restoreOld()
defer restoreNew()
_ = os.Setenv(oldSecret, "disposable-old-example-credential")
_ = os.Setenv(newSecret, "disposable-new-example-credential")
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), "upsert", "preview")
if err != nil {
fmt.Println(err)
return
}
ids := make([]string, 0, len(result.Config.Prompts))
for _, prompt := range result.Config.Prompts {
ids = append(ids, prompt.ID)
}
shared := configkit.Keys("prompts").Index(1).Key("name")
info, _ := result.Metadata().Lookup(shared)
resource, resourceErr := consumers.ReadSequenceCatalogResource(result, "concise")
fmt.Println(ids, result.Config.ResolvedLabel, info.Explicit, len(result.Metadata().History(shared)), resource, resourceErr == nil, err)
// Output: [foundation shared welcome concise extra] for:preview true 2 Foundation resource text.
// true <nil>
}
func TestSequenceCatalogSelectedOverlayOperations(t *testing.T) {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
t.Setenv(oldSecret, "sequence-catalog-old-canary")
t.Setenv(newSecret, "sequence-catalog-new-canary")
tests := []struct {
name string
wantIDs []string
shared *consumers.SequenceCatalogEntry
}{
{name: "append", wantIDs: []string{"foundation", "shared", "welcome", "concise", "extra"}},
{name: "upsert", wantIDs: []string{"foundation", "shared", "welcome", "concise", "extra"}, shared: &consumers.SequenceCatalogEntry{ID: "shared", Name: "Shared replacement", Messages: []string{"Replacement message"}, Credential: configkit.SecretEnv(newSecret)}},
{name: "replace", wantIDs: []string{"replacement", "last"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), test.name, "test")
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("load selected overlay: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
gotIDs := make([]string, 0, len(result.Config.Prompts))
for _, prompt := range result.Config.Prompts {
gotIDs = append(gotIDs, prompt.ID)
}
if strings.Join(gotIDs, ",") != strings.Join(test.wantIDs, ",") {
t.Fatalf("prompt order = %v, want %v", gotIDs, test.wantIDs)
}
if test.shared == nil {
return
}
shared := result.Config.Prompts[1]
if shared.ID != test.shared.ID || shared.Name != test.shared.Name || len(shared.Messages) != 1 ||
shared.Messages[0] != test.shared.Messages[0] || shared.Description != "" || shared.Resource != "" || shared.Credential.EnvName() != newSecret {
t.Fatalf("upsert did not replace the complete matched record: %#v", shared)
}
})
}
}
func TestSequenceCatalogInspectionTracksResourcesAndSecretHistory(t *testing.T) {
const oldSecret = "CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET"
const newSecret = "CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET"
const oldCanary = "sequence-catalog-old-canary"
const newCanary = "sequence-catalog-new-canary"
t.Setenv(oldSecret, oldCanary)
t.Setenv(newSecret, newCanary)
result, err := consumers.LoadSequenceCatalog(fixture("sequence-catalog", "config.yml"), "upsert", "history")
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("load sequence catalog: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
resourcePath := configkit.Keys("prompts").Index(3).Key("resource")
resourceInfo, found := result.Metadata().Lookup(resourcePath)
if !found || len(resourceInfo.Origin.Locations) != 1 || len(resourceInfo.Origin.Inheritance) != 1 {
t.Fatalf("inherited resource inspection = %#v, %v", resourceInfo, found)
}
resource, err := consumers.ReadSequenceCatalogResource(result, "concise")
if err != nil || resource != "Foundation resource text.\n" {
t.Fatalf("resource from inherited source context = %q, %v", resource, err)
}
if len(result.Metadata().History(configkit.Keys("resolved_label"))) == 0 {
t.Fatal("recorded CLI-driven resolution was absent from final inspection")
}
credentialPath := configkit.Keys("prompts").Index(1).Key("credential")
credentialHistory := result.Metadata().History(credentialPath)
if len(credentialHistory) == 0 {
t.Fatal("upsert did not retain the replaced secret reference in history")
}
var explanation bytes.Buffer
if err := result.WriteExplanation(&explanation, credentialPath); err != nil {
t.Fatal(err)
}
if strings.Contains(explanation.String(), oldCanary) || strings.Contains(explanation.String(), newCanary) {
t.Fatalf("secret explanation exposed a credential value: %s", explanation.String())
}
}
func TestNamespacedTypedMapIdentityScopesAreIndependent(t *testing.T) {
source := configkit.Bytes("namespaced maps", []byte(`
prompts:
- id: shared
text: root prompt
collections:
team:
prompts:
- id: shared
text: team prompt
operations:
prompts:
- id: shared
text: operations prompt
`))
result, err := consumers.LoadNamespacedMapCatalog(source)
if err != nil {
t.Fatal(err)
}
if result.Config.Prompts[0]["text"] != "root prompt" ||
result.Config.Collections["team"]["prompts"].([]any)[0].(map[string]any)["text"] != "team prompt" ||
result.Config.Collections["operations"]["prompts"].([]any)[0].(map[string]any)["text"] != "operations prompt" {
t.Fatalf("typed map collections = %#v", result.Config)
}
}
func TestNamespacedTypedMapIdentityFindsCollectionsIntroducedByInheritance(t *testing.T) {
result, err := consumers.LoadNamespacedMapCatalog(configkit.Bytes("inherited namespace", []byte(`
collections:
source:
prompts:
- id: shared
text: inherited prompt
copied:
extends: source
`)))
if err != nil {
t.Fatal(err)
}
copied := result.Config.Collections["copied"]["prompts"].([]any)
if len(copied) != 1 || copied[0].(map[string]any)["text"] != "inherited prompt" {
t.Fatalf("inherited typed-map collection = %#v", result.Config.Collections["copied"])
}
}
func Example_yamlDeclaredIdentityOnly() {
result, err := configkit.Load(configkit.Bytes("YAML identity only", []byte(`
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: initial
`)), configkit.Options[map[string]any]{
Overrides: []configkit.Override{{
Path: configkit.Keys("prompts"),
Value: []any{
map[string]any{"id": "same"},
map[string]any{"id": "same"},
},
Flag: "--prompts",
}},
})
fmt.Println(len(result.Config["prompts"].([]any)), err)
// Output: 2 <nil>
}
func TestYAMLDeclaredIdentityDoesNotRegisterFinalTypedConstraint(t *testing.T) {
result, err := configkit.Load(configkit.Bytes("YAML identity only", []byte(`
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: initial
`)), configkit.Options[map[string]any]{
Overrides: []configkit.Override{{
Path: configkit.Keys("prompts"),
Value: []any{
map[string]any{"id": "same"},
map[string]any{"id": "same"},
},
Flag: "--prompts",
}},
})
if err != nil {
t.Fatalf("YAML-only identity incorrectly became an application constraint: %v", err)
}
prompts := result.Config["prompts"].([]any)
if len(prompts) != 2 || prompts[0].(map[string]any)["id"] != prompts[1].(map[string]any)["id"] {
t.Fatalf("CLI result = %#v", result.Config)
}
}
func TestNamespacedTypedMapRejectsDuplicatesWithinOneCollection(t *testing.T) {
_, err := consumers.LoadNamespacedMapCatalog(configkit.Bytes("duplicate namespace", []byte(`
collections:
team:
prompts:
- id: duplicate
- id: duplicate
`)))
var loadError *configkit.LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 ||
loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict ||
loadError.Diagnostics()[0].Path.String() != "/collections/team/prompts" {
diagnostics := []configkit.Diagnostic(nil)
if loadError != nil {
diagnostics = loadError.Diagnostics()
}
t.Fatalf("duplicate namespaced identity error = %v (%#v)", err, diagnostics)
}
}

View File

@@ -1,4 +1,8 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
imports:
- dir: prompts
exclude: [generated-ignored.yml]

View File

@@ -1,3 +1,8 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
prompts:
- id: invalid
name: Invalid prompt

View File

@@ -0,0 +1,7 @@
run_label: source-default
configkit:
imports:
- dir: fragments
pattern: '*.yml'
destination: [prompts]
placement: sequence-item

View File

@@ -0,0 +1 @@
Foundation resource text.

View File

@@ -0,0 +1,4 @@
id: welcome
name: Welcome
messages: [Welcome message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,3 @@
id: concise
extends: {id: foundation}
name: Concise

View File

@@ -0,0 +1,10 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: append
prompts:
- id: extra
name: Extra
messages: [Extra message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,14 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: replace
prompts:
- id: replacement
name: Replacement
messages: [Replacement message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET
- id: last
name: Last
messages: [Last message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,14 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: upsert
prompts:
- id: shared
name: Shared replacement
messages: [Replacement message]
credential: CONFIGKIT_SEQUENCE_CATALOG_NEW_SECRET
- id: extra
name: Extra
messages: [Extra message]
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

View File

@@ -0,0 +1,18 @@
configkit:
sequences:
- destination: [prompts]
key: id
mode: append
prompts:
- id: foundation
name: Foundation
description: Shared foundation instruction
messages: [Foundation message]
resource: foundation.txt
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET
- id: shared
name: Shared draft
description: This field must not survive a whole-record replacement
messages: [Draft message]
resource: shared.txt
credential: CONFIGKIT_SEQUENCE_CATALOG_OLD_SECRET

316
facade_discovery_test.go Normal file
View File

@@ -0,0 +1,316 @@
package configkit
import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"testing/fstest"
)
type facadeDiscoveryConfig struct {
Value string `yaml:"value"`
Settings map[string]string `yaml:"settings"`
}
type facadeDiscoveryLifecycleConfig struct {
Value string `yaml:"value"`
Resolved string `yaml:"resolved"`
}
type facadeDiscoveryPlacementConfig struct {
Items []facadeDiscoveryPlacementItem `yaml:"items"`
}
type facadeDiscoveryPlacementItem struct {
Name string `yaml:"name"`
}
func TestFacadeSearchPresetUsesCurrentWorkingDirectory(t *testing.T) {
original, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
root := t.TempDir()
if err := os.Chdir(root); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := os.Chdir(original); err != nil {
t.Errorf("restore working directory: %v", err)
}
})
writeFacadeImportFile(t, root, "myapp.yml", "value: working\n")
result, err := Load[facadeDiscoveryConfig](SearchConfig("myapp", WorkingDirectory()))
if err != nil || result.Config.Value != "working" {
t.Fatalf("working-directory search = %#v, %v", result, err)
}
if sources := result.Sources(); len(sources) != 1 || sources[0].Path != canonicalFacadePath(t, filepath.Join(root, "myapp.yml")) {
t.Fatalf("working-directory sources = %#v", sources)
}
}
func TestFacadeSearchCustomGenericNamesAndEmptyFilenameSelection(t *testing.T) {
t.Parallel()
root := t.TempDir()
filename := writeFacadeImportFile(t, root, "config.yml", "value: found\n")
custom := SearchDirectory(root, DirectorySearchOptions{
Filenames: []FilenameGroup{CustomFilenames("config.yml")},
})
result, err := Load[facadeDiscoveryConfig](SearchConfig("", custom))
if err != nil || result.Config.Value != "found" {
t.Fatalf("custom generic-only search with empty slug = %#v, %v", result, err)
}
if sources := result.Sources(); len(sources) != 1 || sources[0].Path != canonicalFacadePath(t, filename) {
t.Fatalf("custom generic-only sources = %#v", sources)
}
defaults := func() facadeDiscoveryConfig { return facadeDiscoveryConfig{Value: "default"} }
empty := SearchDirectory(root, DirectorySearchOptions{Filenames: []FilenameGroup{}})
absent, err := Load(SearchConfig("myapp", empty), Options[facadeDiscoveryConfig]{
Definition: Definition[facadeDiscoveryConfig]{Defaults: defaults},
})
if err != nil || absent.Config.Value != "default" || len(absent.Sources()) != 0 {
t.Fatalf("explicit empty filename selection = %#v, %v", absent, err)
}
omitted, err := Load[facadeDiscoveryConfig](SearchConfig("myapp", SearchDirectory(root)))
if err != nil || omitted.Config.Value != "found" {
t.Fatalf("omitted filename selection did not use defaults = %#v, %v", omitted, err)
}
}
func TestFacadeSearchOrderingRequirednessAndSnapshots(t *testing.T) {
t.Parallel()
root := t.TempDir()
selectedPath := writeFacadeImportFile(t, root, "config.yml", "value: selected\n")
laterPath := writeFacadeImportFile(t, root, "later.yml", "value: later\n")
paths := []string{filepath.Join(root, "missing.yml")}
names := []string{"config.yml"}
groups := []FilenameGroup{CustomFilenames(names...)}
directoryOptions := []DirectorySearchOptions{{Filenames: groups}}
search := Searches(Paths(paths...), SearchDirectory(root, directoryOptions...), Paths(laterPath))
source := SearchConfig("", search, SearchOptions{Required: true})
paths[0] = laterPath
names[0] = "later.yml"
groups[0] = CustomFilenames("later.yml")
directoryOptions[0].Filenames[0] = CustomFilenames("later.yml")
search = Paths(laterPath)
result, err := Load[facadeDiscoveryConfig](source)
if err != nil || result.Config.Value != "selected" {
t.Fatalf("ordered required search after caller mutation = %#v, %v", result, err)
}
if sources := result.Sources(); len(sources) != 1 || sources[0].Path != canonicalFacadePath(t, selectedPath) {
t.Fatalf("ordered search sources = %#v", sources)
}
_, err = Load[facadeDiscoveryConfig](SearchConfig("", Paths(filepath.Join(root, "missing.yml")), SearchOptions{Required: true}))
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("required exhausted search error = %v, want *LoadError", err)
}
foundMissing := false
for _, diagnostic := range loadErr.Diagnostics() {
foundMissing = foundMissing || diagnostic.Code == CodeSourceMissing
}
if !foundMissing {
t.Fatalf("required exhausted search diagnostics = %#v", loadErr.Diagnostics())
}
}
func TestFacadeSearchWorksForLayeredMainAndUnderlay(t *testing.T) {
t.Parallel()
root := t.TempDir()
mainPath := writeFacadeImportFile(t, root, "main.yml", "settings: {main: value}\n")
underlayPath := writeFacadeImportFile(t, root, "underlay/base.yml", "settings: {underlay: value}\n")
result, err := LoadLayered(
Input{Source: SearchConfig("", Paths(mainPath))},
LayeredOptions[facadeDiscoveryConfig]{Underlays: []Underlay{{Inputs: []Input{{
Source: SearchConfig("", SearchDirectory(filepath.Dir(underlayPath), DirectorySearchOptions{
Filenames: []FilenameGroup{CustomFilenames("base.yml")},
})),
}}}}},
)
if err != nil || !reflect.DeepEqual(result.Config.Settings, map[string]string{"main": "value", "underlay": "value"}) {
t.Fatalf("layered search = %#v, %v", result, err)
}
sources := result.Sources()
if len(sources) != 2 || sources[0].Path != canonicalFacadePath(t, mainPath) || sources[0].Layer != LayerBase ||
sources[1].Path != canonicalFacadePath(t, underlayPath) || sources[1].Layer != LayerUnderlay {
t.Fatalf("layered search sources = %#v", sources)
}
}
func TestFacadeSearchSelectedAndAbsentUseRootOnlyContext(t *testing.T) {
t.Parallel()
root := t.TempDir()
mainPath := writeFacadeImportFile(t, root, "main.yml", "value: main\n")
writeFacadeImportFile(t, root, "overlays/testing.yml", "value: overlay\n")
filesystem := &FilesystemContext{Root: root}
options := Options[facadeDiscoveryConfig]{Overlay: "testing", Filesystem: filesystem}
selected, err := Load(SearchConfig("", Paths(mainPath)), options)
if err != nil || selected.Config.Value != "overlay" {
t.Fatalf("selected search with root-only context = %#v, %v", selected, err)
}
sources := selected.Sources()
if len(sources) != 2 || sources[0].Path != canonicalFacadePath(t, mainPath) || sources[0].ID != "source-1" ||
sources[1].ID != "source-2" || sources[1].Layer != LayerOverlay {
t.Fatalf("selected search source evidence = %#v", sources)
}
mainContext, ok := selected.SourceContext("source-1")
if !ok || mainContext.BaseDir != "." {
t.Fatalf("selected file base context = %#v, %v", mainContext, ok)
}
missing := SearchConfig("", Paths(filepath.Join(root, "missing.yml")))
searchedAbsent, err := Load(missing, options)
if err != nil {
t.Fatalf("exhausted search with root-only overlay context: %v", err)
}
explicitAbsent, err := Load(NoSource(), options)
if err != nil {
t.Fatalf("NoSource with root-only overlay context: %v", err)
}
if !reflect.DeepEqual(searchedAbsent.Config, explicitAbsent.Config) ||
!reflect.DeepEqual(searchedAbsent.Sources(), explicitAbsent.Sources()) {
t.Fatalf("searched absence differs from NoSource: %#v / %#v", searchedAbsent, explicitAbsent)
}
left, leftOK := searchedAbsent.Metadata().Lookup(Keys("value"))
right, rightOK := explicitAbsent.Metadata().Lookup(Keys("value"))
if leftOK != rightOK || !reflect.DeepEqual(left, right) || !leftOK {
t.Fatalf("searched absence metadata = %#v, %v; NoSource metadata = %#v, %v", left, leftOK, right, rightOK)
}
if sources := searchedAbsent.Sources(); len(sources) != 1 || sources[0].ID != "source-1" || sources[0].Layer != LayerOverlay {
t.Fatalf("absent search created unexpected source evidence = %#v", sources)
}
}
func TestFacadeSearchRejectsSelectedContextWithoutFallback(t *testing.T) {
t.Parallel()
root := t.TempDir()
inside := writeFacadeImportFile(t, root, "inside.yml", "value: fallback\n")
outsideRoot := t.TempDir()
outside := writeFacadeImportFile(t, outsideRoot, "outside.yml", "value: selected\n")
search := SearchConfig("", Paths(outside, inside))
_, err := Load(search, Options[facadeDiscoveryConfig]{Filesystem: &FilesystemContext{Root: root}})
var loadErr *LoadError
if !errors.As(err, &loadErr) || !hasFacadeDiagnostic(loadErr, CodeImportOutsideRoot) {
t.Fatalf("selected outside-root file error = %v, want containment failure", err)
}
_, err = Load(search, Options[facadeDiscoveryConfig]{Filesystem: &FilesystemContext{BaseDir: root, Root: root}})
if !errors.As(err, &loadErr) || !hasFacadeDiagnostic(loadErr, CodeOptionsInvalid) {
t.Fatalf("selected file with explicit base error = %v, want option failure", err)
}
native := NewFilesystem("native", fstest.MapFS{})
_, err = Load(search, Options[facadeDiscoveryConfig]{Filesystem: &FilesystemContext{FS: &native, BaseDir: ".", Root: "."}})
if !errors.As(err, &loadErr) || !hasFacadeDiagnostic(loadErr, CodeOptionsInvalid) {
t.Fatalf("selected file with native context error = %v, want option failure", err)
}
}
func TestFacadeSearchRunsCallbacksOnceAndKeepsInspectionWithSelectedFile(t *testing.T) {
t.Parallel()
root := t.TempDir()
selectedPath := writeFacadeImportFile(t, root, "selected.yml", "value: selected\n")
writeFacadeImportFile(t, root, "later.yml", "value: later\n")
defaultsCalls, resolveCalls, validateCalls := 0, 0, 0
result, err := Load(SearchConfig("", Paths(selectedPath, filepath.Join(root, "later.yml"))), Options[facadeDiscoveryLifecycleConfig]{
Definition: Definition[facadeDiscoveryLifecycleConfig]{
Defaults: func() facadeDiscoveryLifecycleConfig {
defaultsCalls++
return facadeDiscoveryLifecycleConfig{}
},
Resolve: func(configuration *facadeDiscoveryLifecycleConfig, resolution *Resolution) error {
resolveCalls++
configuration.Resolved = "from:" + configuration.Value
resolution.Record(Keys("resolved"), ChangeNormalization, "derive from selected value", Keys("value"))
return nil
},
Validate: func(_ *facadeDiscoveryLifecycleConfig, _ *Validation) { validateCalls++ },
},
})
if err != nil || result.Config.Value != "selected" || result.Config.Resolved != "from:selected" {
t.Fatalf("search lifecycle result = %#v, %v", result, err)
}
if defaultsCalls != 1 || resolveCalls != 1 || validateCalls != 1 {
t.Fatalf("search callbacks ran defaults=%d resolve=%d validate=%d times", defaultsCalls, resolveCalls, validateCalls)
}
sources := result.Sources()
if len(sources) != 1 || sources[0].Path != canonicalFacadePath(t, selectedPath) || sources[0].ID != "source-1" {
t.Fatalf("selected source evidence = %#v", sources)
}
valueInfo, found := result.Metadata().Lookup(Keys("value"))
if !found || len(valueInfo.Origin.Locations) != 1 || valueInfo.Origin.Locations[0].SourceID != sources[0].ID {
t.Fatalf("selected value metadata = %#v, %v", valueInfo, found)
}
var rendered strings.Builder
if err := result.WriteEffective(&rendered); err != nil || !strings.Contains(rendered.String(), "value: selected") || strings.Contains(rendered.String(), "later") {
t.Fatalf("effective output = %q, %v", rendered.String(), err)
}
}
func TestFacadeSearchSequenceItemAndAbsentInputPlacement(t *testing.T) {
t.Parallel()
root := t.TempDir()
selectedPath := writeFacadeImportFile(t, root, "item.yml", "name: selected\n")
options := Options[facadeDiscoveryPlacementConfig]{
Destination: Keys("items"),
PlacementMode: PlacementModeSequenceItem,
}
selected, err := Load(SearchConfig("", Paths(selectedPath)), options)
if err != nil || len(selected.Config.Items) != 1 || selected.Config.Items[0].Name != "selected" {
t.Fatalf("selected sequence item = %#v, %v", selected, err)
}
if sources := selected.Sources(); len(sources) != 1 || sources[0].Destination.String() != "/items" || sources[0].PlacementMode != PlacementModeSequenceItem {
t.Fatalf("selected sequence-item source = %#v", sources)
}
defaults := func() facadeDiscoveryPlacementConfig {
return facadeDiscoveryPlacementConfig{Items: []facadeDiscoveryPlacementItem{}}
}
absent, err := Load(SearchConfig("", Paths(filepath.Join(root, "missing.yml"))), Options[facadeDiscoveryPlacementConfig]{
Definition: Definition[facadeDiscoveryPlacementConfig]{Defaults: defaults},
Destination: options.Destination,
PlacementMode: options.PlacementMode,
})
if err != nil || len(absent.Config.Items) != 0 || len(absent.Sources()) != 0 {
t.Fatalf("absent search created a sequence item: %#v, %v", absent, err)
}
}
func hasFacadeDiagnostic(err *LoadError, code Code) bool {
for _, diagnostic := range err.Diagnostics() {
if diagnostic.Code == code {
return true
}
}
return false
}
func canonicalFacadePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
canonical, err := filepath.EvalSymlinks(absolute)
if err != nil {
t.Fatal(err)
}
return canonical
}

View File

@@ -0,0 +1,249 @@
package configkit_test
import (
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFinalSequenceIdentityTreatsLateExtendsFieldsAsPayload(t *testing.T) {
for _, phase := range []string{"defaults", "resolution"} {
for _, component := range []any{"application-data", nil} {
for _, duplicate := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/%v/duplicate=%t", phase, component, duplicate), func(t *testing.T) {
items := []any{map[string]any{"extends": component}}
if duplicate {
items = append(items, map[string]any{"extends": component})
}
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "extends")},
Defaults: func() map[string]any {
if phase == "defaults" {
return map[string]any{"items": items}
}
return map[string]any{}
},
}
if phase == "resolution" {
definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["items"] = items
resolution.Record(configkit.Keys("items"), configkit.ChangeNormalization, "supply application payload")
return nil
}
}
loaded, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{Definition: definition})
if duplicate {
loadErr := requireFacadeLoadError(t, err)
diagnostics := loadErr.Diagnostics()
if loaded != nil || len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict || diagnostics[0].Path.String() != "/items" {
t.Fatalf("duplicate effective payload diagnostics = %#v", diagnostics)
}
return
}
if err != nil || loaded == nil {
t.Fatalf("ordinary effective identity was interpreted as a directive: %v", err)
}
value := loaded.Config["items"].([]any)[0].(map[string]any)
if actual, present := value["extends"]; !present || actual != component {
t.Fatalf("effective payload = %#v", value)
}
})
}
}
}
}
func TestFinalSequenceIdentityValidatesLateValues(t *testing.T) {
identity := configkit.SequenceIdentityAt(configkit.Keys("items"), "id")
for _, source := range []string{"defaults", "CLI", "resolution"} {
for _, shape := range []string{"duplicate", "missing", "non-scalar"} {
t.Run(source+"/"+shape, func(t *testing.T) {
items := finalIdentityRecords(shape)
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{identity},
Defaults: func() map[string]any {
if source == "defaults" {
return map[string]any{"items": items}
}
return map[string]any{}
},
}
options := configkit.Options[map[string]any]{Definition: definition}
switch source {
case "CLI":
options.Overrides = []configkit.Override{{Path: configkit.Keys("items"), Value: items, Flag: "--items"}}
case "resolution":
options.Definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["items"] = items
resolution.Record(configkit.Keys("items"), configkit.ChangeNormalization, "create effective records")
return nil
}
}
result, err := configkit.Load(configkit.NoSource(), options)
if result != nil {
t.Fatalf("invalid final identities returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) != 1 {
t.Fatalf("diagnostics = %#v", loadErr.Diagnostics())
}
diagnostic := loadErr.Diagnostics()[0]
wantCode, wantPath := configkit.CodeSequenceInvalid, "/items/0/id"
if shape == "duplicate" {
wantCode, wantPath = configkit.CodeSequenceConflict, "/items"
}
if diagnostic.Code != wantCode || diagnostic.Path.String() != wantPath {
t.Fatalf("diagnostic = %#v, want %s at %s", diagnostic, wantCode, wantPath)
}
if source == "CLI" && (diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-1") {
t.Fatalf("CLI identity diagnostic lost its source: %#v", diagnostic)
}
})
}
}
}
func TestFinalSequenceIdentityMatchesLateDynamicNamespaces(t *testing.T) {
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("items"),
}
for _, source := range []string{"defaults", "CLI", "resolution"} {
t.Run(source, func(t *testing.T) {
name := map[string]string{"defaults": "team", "CLI": "cloud", "resolution": "orbit"}[source]
path := configkit.Keys("collections", name, "items")
items := finalIdentityRecords("duplicate")
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(selector, "id"),
},
Defaults: func() map[string]any {
if source == "defaults" {
return map[string]any{"collections": map[string]any{name: map[string]any{"items": items}}}
}
return map[string]any{}
},
}
options := configkit.Options[map[string]any]{Definition: definition}
switch source {
case "CLI":
options.Overrides = []configkit.Override{{Path: path, Value: items, Flag: "--items"}}
case "resolution":
options.Definition.Resolve = func(value *map[string]any, resolution *configkit.Resolution) error {
(*value)["collections"] = map[string]any{name: map[string]any{"items": items}}
resolution.Record(configkit.Keys("collections"), configkit.ChangeNormalization, "create namespaced records")
return nil
}
}
result, err := configkit.Load(configkit.NoSource(), options)
if result != nil {
t.Fatalf("invalid dynamic identity returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict || diagnostics[0].Path.String() != path.String() {
t.Fatalf("dynamic identity diagnostics = %#v", diagnostics)
}
})
}
}
func TestFinalSequenceIdentityAppliesToLayeredLoads(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Defaults: func() map[string]any {
return map[string]any{"items": finalIdentityRecords("duplicate")}
},
}
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[map[string]any]{Definition: definition})
if result != nil {
t.Fatalf("invalid layered final identities returned a result: %#v", result.Config)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("layered identity diagnostics = %#v", diagnostics)
}
}
func TestFinalSequenceIdentityUsesTypedNumbersAndIgnoresUnmatchedShapes(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("items"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("unmatched"), "id"),
},
}
_, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{
Definition: configkit.Definition[map[string]any]{
SequenceIdentities: definition.SequenceIdentities,
Defaults: func() map[string]any {
return map[string]any{"items": []any{
map[string]any{"id": int64(1)}, map[string]any{"id": float64(1)},
}}
},
},
})
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("typed numeric identity diagnostics = %#v", diagnostics)
}
valid, err := configkit.Load(configkit.Mapping("valid", map[string]any{
"items": []any{
map[string]any{"id": nil}, map[string]any{"id": true},
map[string]any{"id": int64(1)}, map[string]any{"id": "one"},
},
"other": []any{map[string]any{"id": "same"}, map[string]any{"id": "same"}},
}), configkit.Options[map[string]any]{Definition: definition})
if err != nil || valid == nil {
t.Fatalf("valid scalar identities and unmatched duplicate sequence = %#v, %v", valid, err)
}
if _, err := configkit.Load(configkit.Mapping("other-shapes", map[string]any{"items": nil}), configkit.Options[map[string]any]{Definition: definition}); err != nil {
t.Fatalf("null at a registered path was validated as an identity sequence: %v", err)
}
}
func TestFinalSequenceIdentityDiagnosticDoesNotRevealResolvedSecrets(t *testing.T) {
const canary = "FINAL-IDENTITY-CREDENTIAL-CANARY"
const secretName = "CONFIGKIT_FINAL_IDENTITY_SECRET"
t.Setenv(secretName, canary)
type item struct {
ID string `yaml:"id"`
Token configkit.Secret `yaml:"token"`
}
type config struct {
Items []item `yaml:"items"`
}
definition := configkit.Definition[config]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Defaults: func() config {
return config{Items: []item{
{ID: "duplicate", Token: configkit.SecretEnv(secretName)},
{ID: "duplicate", Token: configkit.SecretEnv(secretName)},
}}
},
}
result, err := configkit.Load(configkit.NoSource(), configkit.Options[config]{Definition: definition})
if result != nil {
t.Fatalf("duplicate identity returned a result: %#v", result.Config)
}
if err == nil || strings.Contains(err.Error(), canary) {
t.Fatalf("identity diagnostic leaked a resolved secret: %v", err)
}
loadErr := requireFacadeLoadError(t, err)
if diagnostics := loadErr.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Code != configkit.CodeSequenceConflict {
t.Fatalf("secret-bearing identity diagnostics = %#v", diagnostics)
}
}
func finalIdentityRecords(shape string) []any {
switch shape {
case "duplicate":
return []any{map[string]any{"id": "same"}, map[string]any{"id": "same"}}
case "missing":
return []any{map[string]any{"other": "value"}}
default:
return []any{map[string]any{"id": map[string]any{"nested": "value"}}}
}
}

View File

@@ -40,8 +40,8 @@ entries:
Imports: true,
Filesystem: &FilesystemContext{Root: "bundle", FS: &filesystem},
Definition: Definition[map[string]any]{
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceCompositions: []SequenceComposition{{Path: Keys("services", "records"), IdentityField: "id"}},
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceIdentities: []SequenceIdentity{SequenceIdentityAt(Keys("services", "records"), "id")},
Secret: func(candidate Path) bool {
return candidate.String() == "/services/records/0/version"
},
@@ -105,8 +105,8 @@ entries:
Imports: true,
Filesystem: &FilesystemContext{Root: "bundle", FS: &filesystem},
Definition: Definition[map[string]any]{
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceCompositions: []SequenceComposition{{Path: Keys("services", "records"), IdentityField: "version"}},
Version: &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }},
SequenceIdentities: []SequenceIdentity{SequenceIdentityAt(Keys("services", "records"), "version")},
Secret: func(candidate Path) bool {
return candidate.String() == "/services/records/0/version"
},

View File

@@ -91,6 +91,214 @@ func TestFacadeInheritancePreservesPresenceOriginsAndHistory(t *testing.T) {
}
}
func TestFacadeIdentifiedInheritanceResolvesChainsAndNestedCollections(t *testing.T) {
const secretName = "CONFIGKIT_IDENTIFIED_INHERITANCE_SECRET"
const canary = "IDENTIFIED-INHERITANCE-SECRET-CANARY"
t.Setenv(secretName, canary)
type step struct {
ID string `yaml:"id"`
Value string `yaml:"value"`
}
type job struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Credential configkit.Secret `yaml:"credential"`
Resource string `yaml:"resource"`
Steps []step `yaml:"steps"`
Settings map[string]map[string]string `yaml:"settings"`
}
type configuration struct {
Jobs []job `yaml:"jobs"`
}
input := []byte(`jobs:
- id: child
extends: {id: parent}
name: child-name
- id: parent
extends: {id: base}
- id: base
name: base-name
credential: CONFIGKIT_IDENTIFIED_INHERITANCE_SECRET
resource: icon.txt
steps:
- id: base-step
value: base-step-value
- id: second-step
value: second-step-value
settings:
primary: {value: base-setting}
secondary: {extends: primary}
`)
filesystem := configkit.NewFilesystem("identified-bundle", fstest.MapFS{
"config/identified.yml": &fstest.MapFile{Data: input},
"config/icon.txt": &fstest.MapFile{Data: []byte("identified-parent-resource")},
})
registrations := []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("jobs"), "id"),
}
result, err := configkit.Load(filesystem.File("config/identified.yml"), configkit.Options[configuration]{
Definition: configkit.Definition[configuration]{
SequenceIdentities: registrations,
Secret: func(path configkit.Path) bool {
return path.String() == "/jobs/0/credential" || path.String() == "/jobs/1/credential" || path.String() == "/jobs/2/credential"
},
},
})
if err != nil {
if loadError, ok := err.(*configkit.LoadError); ok {
t.Fatalf("load identified inheritance: %v (%#v)", err, loadError.Diagnostics())
}
t.Fatal(err)
}
if len(result.Config.Jobs) != 3 || result.Config.Jobs[0].ID != "child" || result.Config.Jobs[1].ID != "parent" || result.Config.Jobs[2].ID != "base" {
t.Fatalf("inherited job positions and identities = %#v", result.Config.Jobs)
}
child := result.Config.Jobs[0]
if child.Name != "child-name" || child.Credential.Value() != canary || child.Resource != "icon.txt" || len(child.Steps) != 2 {
t.Fatalf("inherited child = %#v", child)
}
if got := []string{child.Steps[0].ID, child.Steps[1].ID}; strings.Join(got, ",") != "base-step,second-step" {
t.Fatalf("inherited nested steps = %v", got)
}
if child.Steps[0].Value != "base-step-value" || child.Steps[1].Value != "second-step-value" || child.Settings["secondary"]["value"] != "base-setting" {
t.Fatalf("inherited nested collection and mapping = %#v", child)
}
credentialPath := configkit.Keys("jobs").Index(0).Key("credential")
credentialInfo, found := result.Metadata().Lookup(credentialPath)
if !found || len(credentialInfo.Origin.Inheritance) != 2 {
t.Fatalf("inherited secret provenance = %#v, %v", credentialInfo, found)
}
resourceInfo, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("resource"))
if !found || len(resourceInfo.Origin.Inheritance) != 2 || len(resourceInfo.Origin.Locations) != 1 {
t.Fatalf("inherited resource provenance = %#v, %v", resourceInfo, found)
}
context, found := result.SourceContext(resourceInfo.Origin.Locations[0].SourceID)
if !found || context.BaseDir != "config" || context.FS == nil {
t.Fatalf("inherited resource context = %#v, %v", context, found)
}
resource, readErr := fs.ReadFile(context.FS, context.BaseDir+"/"+child.Resource)
if readErr != nil || string(resource) != "identified-parent-resource" {
t.Fatalf("inherited resource = %q, %v", resource, readErr)
}
resourceHistory, found := result.Explain(configkit.Keys("jobs").Index(0).Key("resource"))
if !found || len(resourceHistory.History) == 0 {
t.Fatalf("inherited resource history = %#v, %v", resourceHistory, found)
}
credentialHistory, found := result.Explain(credentialPath)
if !found || !credentialHistory.Effective.Redacted {
t.Fatalf("inherited secret explanation = %#v, %v", credentialHistory, found)
}
identityInfo, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("id"))
if !found || !identityInfo.Explicit || len(identityInfo.Origin.Inheritance) != 0 ||
len(identityInfo.Origin.Locations) != 1 {
t.Fatalf("child identity origin and presence = %#v, %v", identityInfo, found)
}
if _, found := result.Metadata().Lookup(configkit.Keys("jobs").Index(0).Key("extends")); found {
t.Fatal("identified record relationship remained in effective metadata")
}
var rendered bytes.Buffer
if err := result.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), canary) || strings.Contains(rendered.String(), "extends:") {
t.Fatalf("effective inherited configuration = %q, %v", rendered.String(), err)
}
}
func TestFacadeIdentifiedInheritanceFailureDoesNotRevealSecretCanary(t *testing.T) {
const secretName = "CONFIGKIT_IDENTIFIED_INHERITANCE_FAILURE_SECRET"
const canary = "IDENTIFIED-INHERITANCE-FAILURE-CANARY"
t.Setenv(secretName, canary)
type job struct {
ID string `yaml:"id"`
Credential configkit.Secret `yaml:"credential"`
}
type configuration struct {
Jobs []job `yaml:"jobs"`
}
input := []byte(`jobs:
- id: child
extends: {id: missing}
credential: CONFIGKIT_IDENTIFIED_INHERITANCE_FAILURE_SECRET
`)
_, err := configkit.Load(configkit.Bytes("identified-failure.yml", input), configkit.Options[configuration]{
Definition: configkit.Definition[configuration]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("jobs"), "id")},
Secret: func(path configkit.Path) bool { return path.String() == "/jobs/0/credential" },
},
})
if err == nil || strings.Contains(err.Error(), canary) {
t.Fatalf("identified inheritance failure = %v; secret canary must remain undisclosed", err)
}
}
func TestFacadeLayeredIdentifiedInheritanceUsesSurvivingRecords(t *testing.T) {
policy := func(mode string) string {
return "configkit:\n sequences:\n - destination: [jobs]\n key: id\n mode: " + mode + "\n"
}
tests := []struct {
name string
lower string
higher string
wantCode configkit.Code
wantValue string
wantSuccess bool
}{
{
name: "upserted parent supplies its final payload",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: parent, value: higher}, {id: child, extends: {id: parent}}]\n",
wantValue: "higher", wantSuccess: true,
},
{
name: "replacement removes a lower parent before lookup",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("replace") + "jobs: [{id: child, extends: {id: parent}}]\n",
wantCode: configkit.CodeInheritanceMissing,
},
{
name: "upserted child without a directive replaces the invalid lower child",
lower: policy("upsert") + "jobs: [{id: child, extends: {id: missing}, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: child, value: higher}]\n",
wantValue: "higher", wantSuccess: true,
},
{
name: "self-reference cannot resolve through a lower predecessor",
lower: policy("upsert") + "jobs: [{id: parent, value: lower}]\n",
higher: policy("upsert") + "jobs: [{id: parent, extends: {id: parent}, value: higher}]\n",
wantCode: configkit.CodeInheritanceCycle,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("main.yml", []byte(test.higher))}, configkit.LayeredOptions[map[string]any]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower.yml", []byte(test.lower))}}}},
})
if test.wantSuccess {
if err != nil {
t.Fatal(err)
}
jobs, ok := result.Config["jobs"].([]any)
if !ok || len(jobs) == 0 {
t.Fatalf("assembled jobs = %#v", result.Config["jobs"])
}
child, ok := jobs[len(jobs)-1].(map[string]any)
if !ok || child["value"] != test.wantValue {
t.Fatalf("surviving record = %#v, want inherited value %q", jobs[len(jobs)-1], test.wantValue)
}
return
}
loadError, ok := err.(*configkit.LoadError)
if !ok {
t.Fatalf("load error = %v, want code %s", err, test.wantCode)
}
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Code == test.wantCode {
return
}
}
t.Fatalf("load diagnostics = %#v, want code %s", loadError.Diagnostics(), test.wantCode)
})
}
}
func TestFacadeInheritanceLayeredPrecedenceAndSourceContext(t *testing.T) {
filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{
"config/main.yml": &fstest.MapFile{Data: []byte("entries: {child: {extends: parent}}\n")},
@@ -206,27 +414,24 @@ func TestFacadeInheritanceFailureStopsLaterCallbacks(t *testing.T) {
}
}
func TestFacadeInheritanceResolvesAfterImportsAndCLIReferenceChange(t *testing.T) {
func TestFacadeCLIOverridesCannotChangeInheritanceRelationship(t *testing.T) {
filesystem := configkit.NewFilesystem("bundle", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}}\n")},
"parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}, second: {count: 2}}\n")},
})
result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{
_, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[inheritedConfiguration]{
Imports: true, Overrides: []configkit.Override{{Path: configkit.Keys("entries", "child", "extends"), Value: "second", Flag: "--extends"}},
})
if err != nil || result.Config.Entries["child"].Count != 2 {
t.Fatalf("import and CLI reference = %#v, %v", result, err)
}
history := result.Metadata().History(configkit.Keys("entries", "child", "extends"))
if len(history) < 3 || history[len(history)-1].Origin.Operation != configkit.ChangeInheritance {
t.Fatalf("superseded reference history = %#v", history)
loadError, ok := err.(*configkit.LoadError)
if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeOverrideDirective {
t.Fatalf("CLI inheritance relationship change = %v", err)
}
conflicting := configkit.NewFilesystem("bundle", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: parent.yml}]}\nentries: {child: {extends: first}, first: {count: 3}}\n")},
"parent.yml": &fstest.MapFile{Data: []byte("entries: {first: {count: 1}}\n")},
})
_, err = configkit.Load[inheritedConfiguration](conflicting.File("main.yml"), configkit.Options[inheritedConfiguration]{Imports: true})
loadError, ok := err.(*configkit.LoadError)
loadError, ok = err.(*configkit.LoadError)
if !ok || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != configkit.CodeComposeConflict {
t.Fatalf("strict conflict before expansion = %v", err)
}

View File

@@ -0,0 +1,69 @@
package configkit_test
import (
"fmt"
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeItemScopesDoNotReinterpretGlobalRegistrations(t *testing.T) {
t.Parallel()
for _, layered := range []bool{false, true} {
for _, dynamic := range []bool{false, true} {
for _, missingOuterID := range []bool{false, true} {
t.Run(fmt.Sprintf("layered=%t/dynamic=%t/missing-outer-id=%t", layered, dynamic, missingOuterID), func(t *testing.T) {
payload := "prompts: [{text: helper}]\n"
unused := configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id")
if dynamic {
payload = "catalog: {local: {prompts: [{text: helper}]}}\n"
unused = configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("catalog"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts"),
}, "id")
}
item := "configkit: {imports: [{path: helper.yml}, {path: child.yml, destination: [children], placement: sequence-item}]}\n"
if !missingOuterID {
item += "id: outer\n"
}
files := configkit.NewFilesystem("items", fstest.MapFS{
"main.yml": {Data: []byte("configkit: {imports: [{path: item.yml, destination: [items], placement: sequence-item}]}\n")},
"item.yml": {Data: []byte(item)}, "helper.yml": {Data: []byte(payload)},
"child.yml": {Data: []byte("id: child\nconfigkit: {imports: [{path: child-helper.yml}]}\n")},
"child-helper.yml": {Data: []byte(payload)},
})
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
unused, configkit.SequenceIdentityAt(configkit.Keys("items"), "id"),
}}
var result *configkit.Result[map[string]any]
var err error
if layered {
result, err = configkit.LoadLayered(configkit.Input{Source: files.File("main.yml"), Imports: true}, configkit.LayeredOptions[map[string]any]{Definition: definition})
} else {
result, err = configkit.Load(files.File("main.yml"), configkit.Options[map[string]any]{Imports: true, Definition: definition})
}
if missingOuterID {
failure := requireFacadeLoadError(t, err)
if result != nil || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != configkit.CodeComposeInvalid || failure.Diagnostics()[0].Path.String() != "/items/0/id" {
t.Fatalf("outer constraint diagnostics = %#v", failure.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
for _, record := range []configkit.Path{configkit.Keys("items").Index(0), configkit.Keys("items").Index(0).Key("children").Index(0)} {
path := record
if dynamic {
path = path.Key("catalog").Key("local")
}
info, found := result.Metadata().Lookup(path.Key("prompts").Index(0).Key("text"))
if !found || !info.Explicit {
t.Fatalf("nested helper payload lost at %s", path)
}
}
})
}
}
}
}

View File

@@ -0,0 +1,240 @@
package configkit_test
import (
"errors"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFacadeSequenceIdentitySnapshotsSelectorsAndFields(t *testing.T) {
fields := []string{"namespace", "id", "id"}
selector := configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}
identity := configkit.SequenceIdentityMatching(selector, fields...)
selector[1] = configkit.SequenceLiteralKey("other")
fields[0] = "changed"
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{identity}}
underlay := configkit.Mapping("underlay", map[string]any{"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"namespace": "team", "id": "kept"}}},
}})
main := configkit.Mapping("main", map[string]any{"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"namespace": "team", "id": "added"}}},
}})
loaded, err := configkit.LoadLayered(configkit.Input{Source: main}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}}}},
})
if err != nil {
t.Fatalf("load after mutating constructor inputs: %v", err)
}
prompts := loaded.Config["collections"].(map[string]any)["team"].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Fatalf("registered compound identity did not append records: %#v", prompts)
}
}
func TestFacadeSequenceIdentityUsesExactAndDynamicSelectors(t *testing.T) {
dynamic := configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id")
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "namespace", "id"),
dynamic,
}}
underlay := configkit.Mapping("underlay", map[string]any{
"prompts": []any{map[string]any{"namespace": "alpha", "id": "same"}},
"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-one"}}},
"cloud": map[string]any{"prompts": []any{map[string]any{"id": "cloud-one"}}},
},
})
main := configkit.Mapping("main", map[string]any{
"prompts": []any{map[string]any{"namespace": "beta", "id": "same"}},
"collections": map[string]any{
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-two"}}},
"cloud": map[string]any{"prompts": []any{map[string]any{"id": "cloud-two"}}},
},
})
loaded, err := configkit.LoadLayered(configkit.Input{Source: main}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}}}},
})
if err != nil {
t.Fatalf("load mixed exact and dynamic registrations: %v", err)
}
rootRecords := loaded.Config["prompts"].([]any)
if len(rootRecords) != 2 {
t.Fatalf("compound root identities = %#v", rootRecords)
}
collections := loaded.Config["collections"].(map[string]any)
for _, name := range []string{"team", "cloud"} {
prompts := collections[name].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Errorf("dynamic prompts for %q = %#v", name, prompts)
}
}
}
func TestFacadeSequenceIdentityLiteralStarIsNotWildcard(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceLiteralKey("*"),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
underlay := configkit.Mapping("underlay", map[string]any{"collections": map[string]any{
"*": map[string]any{"prompts": []any{map[string]any{"id": "one"}}},
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-one"}}},
}})
main := configkit.Mapping("main", map[string]any{"collections": map[string]any{
"*": map[string]any{"prompts": []any{map[string]any{"id": "two"}}},
"team": map[string]any{"prompts": []any{map[string]any{"id": "team-two"}}},
}})
_, err := configkit.LoadLayered(configkit.Input{Source: configkit.Mapping("main", map[string]any{})}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}, {Source: main}}}},
})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Path.String() != "/collections/team/prompts" {
t.Fatalf("literal star registration also matched another key: %#v", loadErr.Diagnostics())
}
wildcard := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
loaded, err := configkit.LoadLayered(configkit.Input{Source: configkit.Mapping("main", map[string]any{})}, configkit.LayeredOptions[map[string]any]{
Definition: wildcard,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: underlay}, {Source: main}}}},
})
if err != nil {
t.Fatalf("wildcard registration did not match mapping keys: %v", err)
}
for name, raw := range loaded.Config["collections"].(map[string]any) {
if got := len(raw.(map[string]any)["prompts"].([]any)); got != 2 {
t.Errorf("wildcard result for %q has %d records", name, got)
}
}
}
func TestFacadeSequenceIdentityRejectsInvalidDefinitionsBeforeReadingSource(t *testing.T) {
tests := []struct {
name string
identity configkit.SequenceIdentity
wantPath string
}{
{
name: "index path",
identity: configkit.SequenceIdentityAt(configkit.Keys("records").Index(0), "id"),
wantPath: "/records/0",
},
{
name: "empty fields",
identity: configkit.SequenceIdentityAt(configkit.Keys("records")),
wantPath: "/records",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
reader := strings.NewReader("records: []\n")
_, err := configkit.Load(configkit.Reader("input.yml", reader), configkit.Options[map[string]any]{
Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{test.identity}},
})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != configkit.CodeOptionsInvalid || loadErr.Diagnostics()[0].Path.String() != test.wantPath {
t.Fatalf("definition diagnostics = %#v", loadErr.Diagnostics())
}
if reader.Len() == 0 {
t.Fatal("source was read before registration validation")
}
})
}
}
func TestFacadeSequenceIdentityOverlapFailsBeforeSourceAccess(t *testing.T) {
selector := configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey()}
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(selector, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"), configkit.SequenceLiteralKey("team"),
}, "slug"),
}}
reader := strings.NewReader("collections: {}\n")
_, err := configkit.Load(configkit.Reader("input.yml", reader), configkit.Options[map[string]any]{Definition: definition})
loadErr := requireFacadeLoadError(t, err)
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != configkit.CodeOptionsInvalid {
t.Fatalf("overlap diagnostics = %#v", loadErr.Diagnostics())
}
if reader.Len() == 0 {
t.Fatal("source was read before overlapping registrations were rejected")
}
}
func TestFacadeSequenceIdentityConstraintsApplyAtInheritedDestinations(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("entries"),
configkit.SequenceLiteralKey("child"),
configkit.SequenceLiteralKey("collections"),
configkit.SequenceAnyMappingKey(),
configkit.SequenceLiteralKey("prompts"),
}, "id"),
}}
loaded, err := configkit.Load(configkit.Bytes("main", []byte(`jobs:
entries:
parent:
collections:
team:
prompts: [{id: inherited, text: base}]
child:
extends: parent
collections:
team:
prompts: [{id: added, text: child}]
`)), configkit.Options[map[string]any]{Definition: definition})
if err != nil {
t.Fatalf("load with dynamic inherited destination: %v", err)
}
child := loaded.Config["entries"].(map[string]any)["child"].(map[string]any)
prompts := child["collections"].(map[string]any)["team"].(map[string]any)["prompts"].([]any)
if len(prompts) != 2 {
t.Fatalf("inherited dynamic collection = %#v, want parent and child records", prompts)
}
}
func TestFacadeSequenceIdentityLoadsUseIndependentScopes(t *testing.T) {
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityMatching(configkit.SequenceSelector{
configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(),
}, "id"),
}}
for _, source := range []string{"first", "second"} {
loaded, err := configkit.Load(configkit.Mapping(source, map[string]any{"collections": map[string]any{
source: []any{map[string]any{"id": "reused"}},
}}), configkit.Options[map[string]any]{Definition: definition})
if err != nil || len(loaded.Config["collections"].(map[string]any)[source].([]any)) != 1 {
t.Fatalf("independent load %q = %#v, %v", source, loaded, err)
}
}
}
func requireFacadeLoadError(t *testing.T, err error) *configkit.LoadError {
t.Helper()
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("error = %v, want *LoadError", err)
}
return loadErr
}

View File

@@ -21,7 +21,7 @@ func TestFacadeNoSourcePreservesAbsenceWithSequencePolicy(t *testing.T) {
definition.Defaults = func() map[string]any { return map[string]any{"fallback": true} }
}
if registered {
definition.SequenceCompositions = []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}
definition.SequenceIdentities = []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}
}
ordinary, err := configkit.Load(configkit.NoSource(), configkit.Options[map[string]any]{Definition: definition})
if err != nil {
@@ -63,7 +63,7 @@ func TestFacadeItemHelperHistoryPreservesScheduledPlacement(t *testing.T) {
"child-helper.yml": {Data: []byte("text: nested helper\n")},
})
result, err := configkit.Load(filesystem.File("main.yml"), configkit.Options[map[string]any]{
Imports: true, Definition: configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}},
Imports: true, Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}},
})
if err != nil {
t.Fatal(err)
@@ -107,8 +107,14 @@ func TestFacadeRegisteredReplacementExplainsRemovedDescendants(t *testing.T) {
{"removed field", []any{map[string]any{"id": "main"}}, configkit.Keys("records").Index(0).Key("text")},
} {
t.Run(test.name, func(t *testing.T) {
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Mapping("main", map[string]any{"records": test.upper})}, configkit.LayeredOptions[map[string]any]{
Definition: configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}},
mainYAML := "configkit:\n sequences:\n - destination: [records]\n key: id\n mode: replace\n"
if len(test.upper) == 0 {
mainYAML += "records: []\n"
} else {
mainYAML += "records: [{id: main}]\n"
}
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte(mainYAML))}, configkit.LayeredOptions[map[string]any]{
Definition: configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("lower", map[string]any{"records": []any{map[string]any{"id": "one", "text": "old"}, map[string]any{"id": "two"}}})}}}},
})
if err != nil {
@@ -137,7 +143,7 @@ func TestFacadeItemConflictDiagnosticsFollowEnclosingIndexes(t *testing.T) {
"bad.yml": {Data: []byte("id: bad\nconfigkit: {imports: [{path: helper.yml}]}\n")},
"helper.yml": {Data: []byte("id: duplicate-field\n")},
})
definition := configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}}
definition := configkit.Definition[map[string]any]{SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")}}
for _, test := range []struct {
name, file, path string
grouped, itemMain bool
@@ -194,7 +200,7 @@ func TestFacadeItemConflictDiagnosticsFollowEnclosingIndexes(t *testing.T) {
func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("entries"), IdentityField: "id"}},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
}
ordinary, err := configkit.Load(configkit.Mapping("ordinary", map[string]any{"id": "one"}), configkit.Options[map[string]any]{
Definition: definition, Destination: configkit.Keys("entries"), PlacementMode: configkit.PlacementModeSequenceItem,
@@ -240,8 +246,8 @@ func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) {
if err != nil {
t.Fatalf("registered sequence precedence load: %v", err)
}
if got := precedence.Config["entries"].([]any); len(got) != 1 || got[0].(map[string]any)["id"] != "main" {
t.Fatalf("higher-priority list did not replace the lower sequence: %#v", precedence.Config)
if got := precedence.Config["entries"].([]any); len(got) != 2 || got[0].(map[string]any)["id"] != "lower" || got[1].(map[string]any)["id"] != "main" {
t.Fatalf("registered upsert did not retain the lower sequence: %#v", precedence.Config)
}
for _, test := range []struct {
@@ -295,7 +301,7 @@ items:
result, err := configkit.Load(configkit.File(mainPath), configkit.Options[map[string]any]{
Imports: true,
Definition: configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("items"), IdentityField: "id"}},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Resolve: func(_ *map[string]any, resolution *configkit.Resolution) error {
for _, id := range []string{"source-2", "source-3"} {
source, ok := resolution.Source(id)
@@ -455,15 +461,17 @@ func TestFacadeNestedItemFallbackUsesNearestResolvedContainer(t *testing.T) {
}
}
func TestFacadeSequenceCompositionRegistrationPathIsDetached(t *testing.T) {
registrations := []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}
func TestFacadeSequenceIdentityRegistrationIsDetached(t *testing.T) {
fields := []string{"id"}
path := configkit.Keys("records")
registration := configkit.SequenceIdentityAt(path, fields...)
fields[0] = "changed"
definition := configkit.Definition[map[string]any]{
SequenceCompositions: registrations,
SequenceIdentities: []configkit.SequenceIdentity{registration},
Version: &configkit.VersionRule{
Path: configkit.Keys("version"),
Check: func(configkit.Version, *configkit.Validation) error {
registrations[0].Path = configkit.Keys("other")
registrations[0].IdentityField = "changed"
path = configkit.Keys("other")
return nil
},
},
@@ -506,6 +514,31 @@ func TestFacadeSourceInfoPlacementCopiesAndZeroValues(t *testing.T) {
}
}
func TestLayeredSequenceUpsertHistoryRetainsLowerTierRecords(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
}
result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte("entries: [{id: main}]\n"))}, configkit.LayeredOptions[map[string]any]{
Definition: definition,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: configkit.Bytes("lower", []byte("entries: [{id: lower}]\n")),
}}}},
})
if err != nil {
t.Fatal(err)
}
for index, id := range []string{"lower", "main"} {
path := configkit.Keys("entries").Index(index).Key("id")
explanation, ok := result.Explain(path)
if !ok || !explanation.Effective.Present || explanation.Effective.YAML != id+"\n" || len(explanation.History) == 0 {
t.Fatalf("tier history at %s = %#v, %v", path.String(), explanation, ok)
}
if explanation.History[len(explanation.History)-1].After.YAML != id+"\n" {
t.Fatalf("tier history at %s does not retain final record: %#v", path.String(), explanation.History)
}
}
}
func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *testing.T) {
type record struct {
ID string `yaml:"id"`
@@ -523,7 +556,12 @@ func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *test
directory := t.TempDir()
mainPath := filepath.Join(directory, "main.yml")
underlayPath := filepath.Join(directory, "underlay.yml")
if err := os.WriteFile(mainPath, []byte(`entries:
if err := os.WriteFile(mainPath, []byte(`configkit:
sequences:
- destination: [entries]
key: id
mode: replace
entries:
- id: current
count: 0
enabled: false
@@ -559,7 +597,7 @@ func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *test
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.File(mainPath)}, configkit.LayeredOptions[configuration]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.File(underlayPath), Imports: true}}}},
Definition: configkit.Definition[configuration]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("entries"), IdentityField: "id"}},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("entries"), "id")},
Secret: func(path configkit.Path) bool {
return path.String() == "/entries/0/token" || path.String() == "/entries/1/token"
},

View File

@@ -0,0 +1,197 @@
package configkit_test
import (
"fmt"
"testing"
"testing/fstest"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
type movedVersionItem struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
}
type movedVersionCatalog struct {
Items []movedVersionItem `yaml:"items"`
}
func TestMainVersionDecodingFollowsSequenceMovement(t *testing.T) {
t.Parallel()
for _, mounted := range []bool{false, true} {
for _, matching := range []bool{false, true} {
for _, version := range []string{"4", "'4'"} {
t.Run(fmt.Sprintf("mounted=%t/matched=%t/%s", mounted, matching, version), func(t *testing.T) {
body := "items: [{id: main, version: " + version + "}]\n"
path := configkit.Keys("items").Index(0).Key("version")
main := configkit.Input{}
if mounted {
body, path = "id: main\nversion: "+version+"\n", configkit.Keys("version")
main.Destination, main.PlacementMode = configkit.Keys("items"), configkit.PlacementModeSequenceItem
}
main.Source = configkit.Bytes("main", []byte(body))
keep := "{id: keep}"
if mounted {
keep = "{id: keep, version: ordinary}"
}
lower := "items: [" + keep + "]\n"
if matching {
lower = "items: [" + keep + ", {id: main}]\n"
}
calls := 0
loaded, err := configkit.LoadLayered(main, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower", []byte(lower))}}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: path, Check: func(version configkit.Version, _ *configkit.Validation) error {
calls++
if !version.Present || version.Text != "4" {
t.Errorf("checked version = %#v", version)
}
return nil
}},
},
})
if err != nil {
t.Fatal(err)
}
if calls != 1 || len(loaded.Config.Items) != 2 || loaded.Config.Items[1].ID != "main" || loaded.Config.Items[1].Version != "4" {
t.Fatalf("moved main = %#v, checker calls = %d", loaded.Config, calls)
}
if mounted && loaded.Config.Items[0].Version != "ordinary" {
t.Fatal("unrelated underlay payload was treated as the main declaration")
}
for _, source := range loaded.Sources() {
if source.Layer == configkit.LayerBase && source.VersionCheck.Path.String() != path.String() {
t.Fatal("effective movement changed source-local version evidence")
}
}
})
}
}
}
}
func TestOverlayVersionProtectionUsesAssignedRecordPositions(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name, payload string
code configkit.Code
}{
{name: "change main header", payload: "[{id: main, version: changed}]", code: configkit.CodeOverlayVersion},
{name: "remove main header", payload: "[{id: main}]", code: configkit.CodeOverlayVersion},
{name: "other record version", payload: "[{id: keep, version: ordinary}]"},
{name: "new record version", payload: "[{id: new, version: ordinary}]"},
{name: "only main header gets integer conversion", payload: "[{id: new, version: 5}]", code: configkit.CodeDecodeType},
} {
t.Run(test.name, func(t *testing.T) {
files := configkit.NewFilesystem("catalog", fstest.MapFS{
"overlays/test.yml": {Data: []byte("items: " + test.payload + "\n")},
})
calls := 0
loaded, err := configkit.LoadLayered(configkit.Input{
Source: configkit.Bytes("main", []byte("id: main\nversion: 4\n")), Destination: configkit.Keys("items"), PlacementMode: configkit.PlacementModeSequenceItem,
}, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Overlay: "test", OverlayFilesystem: &configkit.FilesystemContext{FS: &files, BaseDir: ".", Root: "."},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("lower", []byte("items: [{id: keep}]\n"))}}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
if calls != 1 {
t.Fatalf("checker calls = %d", calls)
}
if test.code != "" {
loadErr := requireFacadeLoadError(t, err)
if loaded != nil || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != test.code {
t.Fatalf("version diagnostics = %#v", loadErr.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
if loaded.Config.Items[1].Version != "4" {
t.Fatal("unrelated overlay changed main header")
}
})
}
}
func TestUnderlayVersionProtectionUsesMainRecordPosition(t *testing.T) {
t.Parallel()
for _, imported := range []bool{false, true} {
t.Run(fmt.Sprintf("imported=%t", imported), func(t *testing.T) {
payload := "items: [{id: keep}, {id: main, version: forbidden}]\n"
lower := configkit.Input{Source: configkit.Bytes("lower", []byte(payload))}
want := configkit.CodeUnderlayVersion
if imported {
files := configkit.NewFilesystem("lower", fstest.MapFS{
"main.yml": {Data: []byte("configkit: {imports: [{path: fragment.yml}]}\n")},
"fragment.yml": {Data: []byte(payload)},
})
lower.Source, lower.Imports = files.File("main.yml"), true
want = configkit.CodeImportVersion
}
calls := 0
result, err := configkit.LoadLayered(configkit.Input{
Source: configkit.Bytes("main", []byte("id: main\nversion: 4\n")),
Destination: configkit.Keys("items"), PlacementMode: configkit.PlacementModeSequenceItem,
}, configkit.LayeredOptions[movedVersionCatalog]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{lower}}},
Definition: configkit.Definition[movedVersionCatalog]{
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("items"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
failure := requireFacadeLoadError(t, err)
if result != nil || calls != 1 || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != want || failure.Diagnostics()[0].Path.String() != "/items/1/version" {
t.Fatalf("result=%#v calls=%d diagnostics=%#v", result, calls, failure.Diagnostics())
}
})
}
}
func TestMainVersionFollowsInheritedSequenceMovement(t *testing.T) {
t.Parallel()
for _, parentID := range []string{"keep", "main"} {
t.Run(parentID, func(t *testing.T) {
body := "base: {items: [{id: " + parentID + ", version: ordinary}]}\nchild:\n extends: base\n items: [{id: main, version: 4}]\n"
type config struct {
Base, Child movedVersionCatalog
}
calls := 0
result, err := configkit.Load(configkit.Bytes("main", []byte(body)), configkit.Options[config]{
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Definition: configkit.Definition[config]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("base", "items"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("child", "items"), "id"),
},
Version: &configkit.VersionRule{Path: configkit.Keys("child", "items").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { calls++; return nil }},
},
})
if calls != 1 {
t.Fatalf("checker calls = %d", calls)
}
if parentID == "main" {
failure := requireFacadeLoadError(t, err)
if result != nil || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != configkit.CodeInheritanceVersion {
t.Fatalf("diagnostics = %#v", failure.Diagnostics())
}
return
}
if err != nil {
t.Fatal(err)
}
if len(result.Config.Child.Items) != 2 || result.Config.Child.Items[1].Version != "4" || result.Config.Child.Items[0].Version != "ordinary" {
t.Fatalf("inherited items = %#v", result.Config.Child.Items)
}
})
}
}

View File

@@ -23,9 +23,9 @@ func TestFacadeSourceVersionEvidenceFollowsAssembledSequenceRanges(t *testing.T)
for _, sensitivePath := range []string{"", fmt.Sprintf("/records/%d/version", index), fmt.Sprintf("/records/%d", index)} {
t.Run(fmt.Sprintf("grouped=%t/secret=%s", grouped, sensitivePath), func(t *testing.T) {
definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}},
Version: &configkit.VersionRule{Path: configkit.Keys("records").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { return nil }},
Secret: func(path configkit.Path) bool { return sensitivePath != "" && path.String() == sensitivePath },
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")},
Version: &configkit.VersionRule{Path: configkit.Keys("records").Index(0).Key("version"), Check: func(configkit.Version, *configkit.Validation) error { return nil }},
Secret: func(path configkit.Path) bool { return sensitivePath != "" && path.String() == sensitivePath },
}
first := "records: [{id: first, version: first-version}]\n"
child := "records: [{id: child, version: PRIVATE-VERSION-CANARY}]\n"
@@ -86,8 +86,8 @@ func TestFacadeSourceVersionOwnershipAcceptsContextualIndexes(t *testing.T) {
for _, grouped := range []bool{false, true} {
for _, path := range []configkit.Path{configkit.Keys("records").Index(0).Key("version"), parsed} {
definition := configkit.Definition[map[string]any]{
SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}},
Version: &configkit.VersionRule{Path: path, Check: func(configkit.Version, *configkit.Validation) error { return nil }},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("records"), "id")},
Version: &configkit.VersionRule{Path: path, Check: func(configkit.Version, *configkit.Validation) error { return nil }},
}
first := "records: [{id: first, version: first}]\n"
child := "records: [{id: child, version: child}]\n"
@@ -104,11 +104,7 @@ func TestFacadeSourceVersionOwnershipAcceptsContextualIndexes(t *testing.T) {
if err != nil || result == nil {
t.Fatalf("grouped=%t path=%#v: %v", grouped, path, err)
}
want := 2
if grouped {
want = 1
}
if len(result.Config["records"].([]any)) != want {
if len(result.Config["records"].([]any)) != 2 {
t.Fatal("version path changed composition")
}
}

View File

@@ -0,0 +1,106 @@
package engine
import (
"errors"
"fmt"
"testing"
"testing/fstest"
)
func TestNestedCollectionInitializationIsIndependentOfAncestorPresence(t *testing.T) {
t.Parallel()
for _, ancestor := range []string{"{}", "group: {}", "group: null"} {
for _, test := range []struct {
mode, key, value string
valid bool
}{
{mode: "append", value: "[{id: a}]"},
{mode: "upsert", value: "[{id: a}]"},
{mode: "append", value: "[]"},
{mode: "upsert", value: "[]"},
{mode: "append", key: ", key: id", value: "{}"},
{mode: "upsert", key: ", key: id", value: "null"},
{mode: "append", key: ", key: id", value: "[{id: a}]", valid: true},
{mode: "upsert", key: ", key: id", value: "[]", valid: true},
} {
t.Run(fmt.Sprintf("%s/%s/%s/%s", ancestor, test.mode, test.key, test.value), func(t *testing.T) {
main := "configkit:\n sequences: [{destination: [group, items], mode: " + test.mode + test.key + "}]\ngroup: {items: " + test.value + "}\n"
loaded, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(ancestor))}}}},
})
if test.valid {
if err != nil || loaded == nil {
t.Fatalf("valid initialization failed: %v", err)
}
return
}
var loadErr *LoadError
if !errors.As(err, &loadErr) || loaded != nil {
t.Fatalf("invalid collection operation succeeded: %#v, %v", loaded, err)
}
diagnostics := loadErr.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/group/items" {
t.Fatalf("operation diagnostics = %#v", diagnostics)
}
})
}
}
}
func TestAddedAncestorTransfersNestedRecordPolicies(t *testing.T) {
t.Parallel()
type record struct {
ID string `yaml:"id"`
Steps []map[string]any `yaml:"steps"`
}
type config struct {
Group struct {
Items []record `yaml:"items"`
} `yaml:"group"`
}
files := fstest.MapFS{
"main.yml": {Data: []byte(`configkit:
sequences: [{destination: [group, items], key: id}]
imports: [{path: base.yml, destination: [group, items], placement: sequence-item}]
group:
items: [{id: child, extends: {id: base}, steps: [{name: child}]}]
`)},
"base.yml": {Data: []byte("configkit:\n sequences: [{destination: [steps], key: name}]\nid: base\nsteps: [{name: base}]\n")},
}
loaded, err := testGroupedLoad(loadRequest[config]{
main: loadInput{source: NewFilesystem("catalog", files).File("main.yml"), imports: true},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("{}"))}}}},
})
if err != nil {
t.Fatal(err)
}
for _, item := range loaded.Config.Group.Items {
if item.ID == "child" {
if len(item.Steps) != 2 || item.Steps[0]["name"] != "base" || item.Steps[1]["name"] != "child" {
t.Fatalf("record-local identity was lost when adding its ancestor: %#v", item.Steps)
}
return
}
}
t.Fatal("missing child record")
}
func TestInheritanceChecksCollectionOperationUnderAddedAncestor(t *testing.T) {
t.Parallel()
_, err := testLoad[map[string]any](Bytes("inheritance", []byte(`configkit:
sequences: [{destination: [child, group, items], mode: append}]
base: {}
child:
extends: base
group: {items: [{id: a}]}
`)))
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("missing nested operation error: %v", err)
}
diagnostics := loadErr.Diagnostics()
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/child/group/items" {
t.Fatalf("nested inheritance operation diagnostics = %#v", diagnostics)
}
}

View File

@@ -0,0 +1,98 @@
package engine
import (
"fmt"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestOverlayBroadCollectionPoliciesAndRetainedDeclarations(t *testing.T) {
var baseBody, overlayBody strings.Builder
basePolicies := collectionPolicyScope{}
var declarations []sequenceDeclarationSpec
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
fmt.Fprintf(&baseBody, "%s: {items: [{id: base}], retained: [{id: retained}], scalar: pending}\n", name)
fmt.Fprintf(&overlayBody, "%s: {items: [{id: overlay}], added: true}\n", name)
basePolicies.set(Keys(name, "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
})
for _, key := range []string{"retained", "missing", "scalar"} {
declarations = append(declarations, sequenceDeclarationSpec{
destination: Keys(name, key), identityFields: []string{"id"}, identityPresent: true,
mode: sequenceModeAppend, modePresent: true,
})
}
}
base, diagnostics := composeDocuments(parseCompositionDocument(t, "base", baseBody.String()))
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
base.collectionPolicies = basePolicies
assembly := newAssemblyAccumulator(base)
overlay := parseCompositionDocument(t, "overlay", overlayBody.String())
diagnostics = assembly.applyOverlayWithCollectionPolicy(overlay, nil, collectionPolicyScope{}, nil, LayerOverlay, sequencePolicy{}, declarations)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
lookup := buildDocumentLookup(assembly.current.root)
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
items, found := documentAtPathWithLookup(assembly.current.root, Keys(name, "items"), lookup)
if !found || items.Kind != document.Sequence || len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "overlay" {
t.Fatalf("collection %s did not append in order", name)
}
for _, key := range []string{"retained", "missing", "scalar"} {
policy, found := assembly.current.collectionPolicies.at(Keys(name, key))
if !found || !policy.identityPresent || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
t.Fatalf("retained/pending declaration lost at %s/%s", name, key)
}
}
if _, found := documentAtPathWithLookup(assembly.current.root, Keys(name, "missing"), lookup); found {
t.Fatal("pending declaration created a value")
}
if origin := assembly.current.origins[Keys(name, "items").Index(1).Key("id").String()]; origin.Layer != LayerOverlay {
t.Fatal("appended item lost overlay attribution")
}
}
}
func TestInheritanceBroadCollectionsInOneDefinition(t *testing.T) {
var body strings.Builder
policies := collectionPolicyScope{}
for _, definition := range []string{"base", "child"} {
fmt.Fprintf(&body, "%s:\n", definition)
if definition == "child" {
body.WriteString(" extends: base\n")
}
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
fmt.Fprintf(&body, " %s: {items: [{id: %s}]}\n", name, definition)
policy := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
if definition == "child" {
policy.explicitMode, policy.explicitModePresent = sequenceModeAppend, true
}
policies.set(Keys(definition, name, "items"), policy)
policies.set(Keys(definition, name, "pending"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
}
}
root := parseCompositionDocument(t, "wide-collections", body.String())
result, diagnostics := expandInheritanceWithPolicies(root, nil, policies, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
lookup := buildDocumentLookup(result.root)
for index := range 64 {
name := fmt.Sprintf("group/%03d~", index)
items, found := documentAtPathWithLookup(result.root, Keys("child", name, "items"), lookup)
if !found || len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "child" {
t.Fatalf("inherited collection %s did not append in order", name)
}
if policy, found := result.collectionPolicies.at(Keys("child", name, "pending")); !found || !policy.identityPresent {
t.Fatalf("pending inherited policy lost at %s", name)
}
}
}

View File

@@ -0,0 +1,209 @@
package engine
type collectionPolicy struct {
identityFields []string
identityPresent bool
identityLocation *Location
explicitMode sequenceMode
explicitModePresent bool
modeLocation *Location
recordOrder []*collectionRecordOwner
recordPolicies map[*collectionRecordOwner]collectionPolicyScope
}
type collectionPolicyResolution struct {
policy collectionPolicy
operation sequenceMode
declarations []sequenceDeclarationSpec
declarationsPresent bool
identityDeclarationPresent bool
modeDeclarationPresent bool
applicationIdentityFields []string
applicationIdentityPresent bool
effectiveIdentityFields []string
}
// resolveCollectionPolicy reconciles declarations for one destination before
// peers are composed. The caller supplies only lower policy that is applicable
// to this destination; record survival and ownership are resolved elsewhere.
func resolveCollectionPolicy(
destination Path,
declarations []sequenceDeclarationSpec,
applicationIdentityFields []string,
lower *collectionPolicy,
) (collectionPolicyResolution, []Diagnostic) {
normalizedDeclarations := cloneSequenceDeclarations(declarations)
for index := range normalizedDeclarations {
declaration := &normalizedDeclarations[index]
if !declaration.identityPresent {
continue
}
fields, err := normalizeSequenceIdentityFields(declaration.identityFields)
if err != nil {
return collectionPolicyResolution{}, []Diagnostic{{
Code: CodeSequenceInvalid, Severity: SeverityError, Path: destination.clone(),
Message: "collection identity declaration requires at least one field",
Primary: collectionDeclarationLocation(declaration.identityLocation, declaration.location),
}}
}
declaration.identityFields = fields
}
normalizedApplicationIdentityFields := cloneStrings(applicationIdentityFields)
if len(applicationIdentityFields) != 0 {
var err error
normalizedApplicationIdentityFields, err = normalizeSequenceIdentityFields(applicationIdentityFields)
if err != nil {
return collectionPolicyResolution{}, []Diagnostic{{
Code: CodeOptionsInvalid, Severity: SeverityError, Path: destination.clone(),
Message: "application collection identity fields are invalid",
}}
}
}
result := collectionPolicyResolution{
declarations: normalizedDeclarations,
declarationsPresent: len(normalizedDeclarations) != 0,
applicationIdentityFields: normalizedApplicationIdentityFields,
applicationIdentityPresent: len(normalizedApplicationIdentityFields) != 0,
}
var diagnostics []Diagnostic
var declaredIdentityFields []string
var declaredIdentityLocation *Location
var declaredMode sequenceMode
var declaredModeLocation *Location
for _, declaration := range normalizedDeclarations {
if declaration.identityPresent {
result.identityDeclarationPresent = true
if declaredIdentityFields == nil {
declaredIdentityFields = cloneStrings(declaration.identityFields)
declaredIdentityLocation = collectionDeclarationLocation(declaration.identityLocation, declaration.location)
} else if !equalSequenceIdentityFields(declaredIdentityFields, declaration.identityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(declaration.identityLocation, declaration.location),
"collection identity declarations conflict",
))
}
}
if declaration.modePresent {
result.modeDeclarationPresent = true
if declaredMode == 0 {
declaredMode = declaration.mode
declaredModeLocation = collectionDeclarationLocation(declaration.modeLocation, declaration.location)
} else if declaredMode != declaration.mode {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredModeLocation,
collectionDeclarationLocation(declaration.modeLocation, declaration.location),
"collection operation declarations conflict",
))
}
}
}
if len(diagnostics) != 0 {
return collectionPolicyResolution{}, canonicalDiagnostics(diagnostics)
}
if lower != nil && lower.identityPresent && declaredIdentityFields != nil && !equalSequenceIdentityFields(lower.identityFields, declaredIdentityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(lower.identityLocation, Location{}),
"collection identity declaration conflicts with retained identity",
))
}
if result.applicationIdentityPresent {
if declaredIdentityFields != nil && !equalSequenceIdentityFields(normalizedApplicationIdentityFields, declaredIdentityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation, nil,
"collection identity declaration conflicts with application constraint",
))
}
if lower != nil && lower.identityPresent && !equalSequenceIdentityFields(normalizedApplicationIdentityFields, lower.identityFields) {
diagnostics = append(diagnostics, collectionPolicyConflictDiagnostic(
destination, declaredIdentityLocation,
collectionDeclarationLocation(lower.identityLocation, Location{}),
"retained collection identity conflicts with application constraint",
))
}
}
if len(diagnostics) != 0 {
return collectionPolicyResolution{}, canonicalDiagnostics(diagnostics)
}
if declaredIdentityFields != nil {
result.policy.identityFields = declaredIdentityFields
result.policy.identityPresent = true
result.policy.identityLocation = cloneLocation(declaredIdentityLocation)
} else if lower != nil && lower.identityPresent {
result.policy.identityFields = cloneStrings(lower.identityFields)
result.policy.identityPresent = true
result.policy.identityLocation = cloneLocation(lower.identityLocation)
}
if result.applicationIdentityPresent {
result.effectiveIdentityFields = cloneStrings(normalizedApplicationIdentityFields)
} else if result.policy.identityPresent {
result.effectiveIdentityFields = cloneStrings(result.policy.identityFields)
}
if result.modeDeclarationPresent {
result.policy.explicitMode = declaredMode
result.policy.explicitModePresent = true
result.policy.modeLocation = cloneLocation(declaredModeLocation)
} else if lower != nil && lower.explicitModePresent {
result.policy.explicitMode = lower.explicitMode
result.policy.explicitModePresent = true
result.policy.modeLocation = cloneLocation(lower.modeLocation)
}
result.operation = collectionOperationMode(declaredMode, result.policy.explicitMode, len(result.effectiveIdentityFields) != 0)
return result, nil
}
func collectionOperationMode(current, retained sequenceMode, identified bool) sequenceMode {
if current != 0 {
return current
}
if retained != 0 {
return retained
}
if identified {
return sequenceModeUpsert
}
return sequenceModeReplace
}
func cloneSequenceDeclarations(declarations []sequenceDeclarationSpec) []sequenceDeclarationSpec {
if declarations == nil {
return nil
}
cloned := make([]sequenceDeclarationSpec, len(declarations))
for index, declaration := range declarations {
cloned[index] = declaration
cloned[index].path = declaration.path.clone()
cloned[index].destination = declaration.destination.clone()
cloned[index].identityFields = cloneStrings(declaration.identityFields)
cloned[index].identityLocation = cloneLocation(declaration.identityLocation)
cloned[index].modeLocation = cloneLocation(declaration.modeLocation)
}
return cloned
}
func collectionDeclarationLocation(specific *Location, fallback Location) *Location {
if specific != nil {
return cloneLocation(specific)
}
if fallback != (Location{}) {
return cloneLocation(&fallback)
}
return nil
}
func collectionPolicyConflictDiagnostic(path Path, primary, related *Location, message string) Diagnostic {
diagnostic := Diagnostic{
Code: CodeSequenceConflict, Severity: SeverityError,
Path: path.clone(), Message: message, Primary: cloneLocation(primary),
}
if related != nil {
diagnostic.Related = []Location{*cloneLocation(related)}
}
return diagnostic
}

View File

@@ -0,0 +1,258 @@
package engine
import (
"sort"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
type collectionPolicyScope struct {
collections map[string]collectionPolicyEntry
}
type collectionPolicyEntry struct {
destination Path
policy collectionPolicy
}
type collectionRecordPolicy struct {
owner *collectionRecordOwner
scope collectionPolicyScope
}
// The nonzero field ensures simultaneously live record owners have distinct
// addresses. The token is per-record ownership, not a source or identity key.
type collectionRecordOwner struct {
marker byte
}
type projectedCollectionPolicy struct {
path Path
policy collectionPolicy
}
func newCollectionRecordPolicy() collectionRecordPolicy {
return collectionRecordPolicy{owner: &collectionRecordOwner{marker: 1}}
}
func retainedCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
return collectionRecordPolicy{
owner: record.owner,
scope: cloneCollectionPolicyScope(record.scope),
}
}
func replacedCollectionRecordPolicy() collectionRecordPolicy {
return newCollectionRecordPolicy()
}
func (policy *collectionPolicy) setRecordPolicy(record collectionRecordPolicy) {
if record.owner == nil {
record.owner = newCollectionRecordPolicy().owner
}
if policy.recordPolicies == nil {
policy.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope)
}
if _, exists := policy.recordPolicies[record.owner]; !exists {
policy.recordOrder = append(policy.recordOrder, record.owner)
}
policy.recordPolicies[record.owner] = cloneCollectionPolicyScope(record.scope)
}
func (policy *collectionPolicy) setRecordPolicyAt(index int, record collectionRecordPolicy) {
if index < 0 {
return
}
if record.owner == nil {
record.owner = newCollectionRecordPolicy().owner
}
if policy.recordPolicies == nil {
policy.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope)
}
for len(policy.recordOrder) <= index {
owner := newCollectionRecordPolicy().owner
policy.recordOrder = append(policy.recordOrder, owner)
policy.recordPolicies[owner] = collectionPolicyScope{}
}
delete(policy.recordPolicies, policy.recordOrder[index])
policy.recordOrder[index] = record.owner
policy.recordPolicies[record.owner] = cloneCollectionPolicyScope(record.scope)
}
func (policy collectionPolicy) recordPolicyAt(index int) (collectionRecordPolicy, bool) {
if index < 0 || index >= len(policy.recordOrder) {
return collectionRecordPolicy{}, false
}
owner := policy.recordOrder[index]
scope, ok := policy.recordPolicies[owner]
if !ok {
return collectionRecordPolicy{}, false
}
return collectionRecordPolicy{owner: owner, scope: cloneCollectionPolicyScope(scope)}, true
}
func collectionPolicyForReplacement(lower *collectionPolicy, current collectionPolicy) collectionPolicy {
result := cloneCollectionPolicy(current)
if !current.explicitModePresent && lower != nil && lower.explicitModePresent {
result.explicitMode = lower.explicitMode
result.explicitModePresent = true
result.modeLocation = cloneLocation(lower.modeLocation)
}
return result
}
// collectionPolicyWithoutValue keeps identity and record ownership because no
// collection value was replaced. A key-only declaration can remain pending
// when no sequence is present, or validate the retained sequence immediately.
func collectionPolicyWithoutValue(
lower *collectionPolicy,
current collectionPolicy,
retainedValue *document.Node,
path Path,
) (collectionPolicy, []Diagnostic) {
result := collectionPolicy{}
if lower != nil {
result = cloneCollectionPolicy(*lower)
}
if lower != nil && lower.identityPresent && current.identityPresent && !equalSequenceIdentityFields(lower.identityFields, current.identityFields) {
return collectionPolicy{}, []Diagnostic{collectionPolicyConflictDiagnostic(
path, current.identityLocation, lower.identityLocation,
"collection identity declaration conflicts with retained identity",
)}
}
if current.identityPresent {
result.identityFields = cloneStrings(current.identityFields)
result.identityPresent = true
result.identityLocation = cloneLocation(current.identityLocation)
}
if current.explicitModePresent {
result.explicitMode = current.explicitMode
result.explicitModePresent = true
result.modeLocation = cloneLocation(current.modeLocation)
}
if current.identityPresent && retainedValue != nil && retainedValue.Kind == document.Sequence {
_, diagnostics := validateSequenceIdentity(retainedValue, path, result.identityFields, sequenceIdentityDiagnosticCodes{
invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict,
})
if len(diagnostics) != 0 {
return collectionPolicy{}, diagnostics
}
}
return result, nil
}
func (scope *collectionPolicyScope) set(destination Path, policy collectionPolicy) {
if scope.collections == nil {
scope.collections = make(map[string]collectionPolicyEntry)
}
scope.collections[destination.String()] = collectionPolicyEntry{
destination: destination.clone(), policy: cloneCollectionPolicy(policy),
}
}
func (scope collectionPolicyScope) at(destination Path) (collectionPolicy, bool) {
entry, ok := scope.collections[destination.String()]
if !ok {
return collectionPolicy{}, false
}
return cloneCollectionPolicy(entry.policy), true
}
// retireCollectionPoliciesAtOrBelow clears YAML identity and record-owned state
// below a replaced ancestor while retaining independently scoped explicit modes.
func (scope *collectionPolicyScope) retireCollectionPoliciesAtOrBelow(ancestor Path) {
if scope == nil {
return
}
for key, entry := range scope.collections {
if !pathHasPrefix(entry.destination, ancestor) {
continue
}
clearCollectionIdentity(&entry.policy)
scope.collections[key] = entry
}
}
func projectCollectionPolicyScope(scope collectionPolicyScope, prefix Path) []projectedCollectionPolicy {
keys := make([]string, 0, len(scope.collections))
for key := range scope.collections {
keys = append(keys, key)
}
sort.Strings(keys)
var projected []projectedCollectionPolicy
for _, key := range keys {
entry := scope.collections[key]
concrete := appendDestination(prefix, entry.destination)
projected = append(projected, projectedCollectionPolicy{path: concrete, policy: cloneCollectionPolicy(entry.policy)})
for index, owner := range entry.policy.recordOrder {
if recordScope, ok := entry.policy.recordPolicies[owner]; ok {
projected = append(projected, projectCollectionPolicyScope(recordScope, concrete.Index(index))...)
}
}
}
return projected
}
func cloneCollectionPolicyScope(scope collectionPolicyScope) collectionPolicyScope {
if scope.collections == nil {
return collectionPolicyScope{}
}
cloned := collectionPolicyScope{collections: make(map[string]collectionPolicyEntry, len(scope.collections))}
for key, entry := range scope.collections {
cloned.collections[key] = collectionPolicyEntry{
destination: entry.destination.clone(), policy: cloneCollectionPolicy(entry.policy),
}
}
return cloned
}
func cloneCollectionPolicyScopeForInheritance(scope collectionPolicyScope) collectionPolicyScope {
if scope.collections == nil {
return collectionPolicyScope{}
}
cloned := collectionPolicyScope{collections: make(map[string]collectionPolicyEntry, len(scope.collections))}
for key, entry := range scope.collections {
cloned.collections[key] = collectionPolicyEntry{
destination: entry.destination.clone(), policy: cloneCollectionPolicyForInheritance(entry.policy),
}
}
return cloned
}
func cloneCollectionPolicy(policy collectionPolicy) collectionPolicy {
cloned := policy
cloned.identityFields = cloneStrings(policy.identityFields)
cloned.identityLocation = cloneLocation(policy.identityLocation)
cloned.modeLocation = cloneLocation(policy.modeLocation)
if policy.recordOrder != nil {
cloned.recordOrder = append([]*collectionRecordOwner(nil), policy.recordOrder...)
}
if policy.recordPolicies != nil {
cloned.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope, len(policy.recordPolicies))
for owner, scope := range policy.recordPolicies {
cloned.recordPolicies[owner] = cloneCollectionPolicyScope(scope)
}
}
return cloned
}
func cloneCollectionPolicyForInheritance(policy collectionPolicy) collectionPolicy {
cloned := policy
cloned.identityFields = cloneStrings(policy.identityFields)
cloned.identityLocation = cloneLocation(policy.identityLocation)
cloned.modeLocation = cloneLocation(policy.modeLocation)
if policy.recordOrder != nil {
cloned.recordOrder = make([]*collectionRecordOwner, 0, len(policy.recordOrder))
}
if policy.recordPolicies != nil {
cloned.recordPolicies = make(map[*collectionRecordOwner]collectionPolicyScope, len(policy.recordPolicies))
for _, sourceOwner := range policy.recordOrder {
owner := newCollectionRecordPolicy().owner
cloned.recordOrder = append(cloned.recordOrder, owner)
if sourceScope, ok := policy.recordPolicies[sourceOwner]; ok {
cloned.recordPolicies[owner] = cloneCollectionPolicyScopeForInheritance(sourceScope)
}
}
}
return cloned
}

View File

@@ -0,0 +1,292 @@
package engine
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestCollectionPolicyWithoutValueRetainsIdentityAndMode(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
current := collectionPolicy{explicitMode: sequenceModeReplace, explicitModePresent: true}
retained, diagnostics := collectionPolicyWithoutValue(&lower, current, nil, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent || !reflect.DeepEqual(retained.identityFields, []string{"id"}) {
t.Fatalf("no-value replace reset identity: %#v, %#v", retained, diagnostics)
}
if retained.explicitMode != sequenceModeReplace || !retained.explicitModePresent || len(retained.recordOrder) != 1 {
t.Fatalf("no-value policy update lost mode or record ownership: %#v", retained)
}
retainedRecord, _ := retained.recordPolicyAt(0)
lowerRecord, _ := lower.recordPolicyAt(0)
if retainedRecord.owner != lowerRecord.owner {
t.Fatal("no-value policy update changed record ownership")
}
}
func TestCollectionPolicyWithoutValueLeavesAbsentAndNonSequenceDestinationsUnconstrained(t *testing.T) {
current := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
for _, value := range []*document.Node{nil, parsePolicyNode(t, "value: scalar\n").Mapping[0].Value} {
retained, diagnostics := collectionPolicyWithoutValue(nil, current, value, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent || !reflect.DeepEqual(retained.identityFields, []string{"id"}) {
t.Fatalf("pending identity at absent/non-sequence destination = %#v, %#v", retained, diagnostics)
}
}
}
func TestCollectionPolicyWithoutValueValidatesRetainedSequenceImmediately(t *testing.T) {
current := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
valid := parsePolicyNode(t, "items: [{id: one}, {id: two}]\n").Mapping[0].Value
retained, diagnostics := collectionPolicyWithoutValue(nil, current, valid, Keys("prompts"))
if len(diagnostics) != 0 || !retained.identityPresent {
t.Fatalf("valid retained records were rejected: %#v, %#v", retained, diagnostics)
}
invalid := parsePolicyNode(t, "items: [{name: missing-id}]\n").Mapping[0].Value
retained, diagnostics = collectionPolicyWithoutValue(nil, current, invalid, Keys("prompts"))
if !reflect.DeepEqual(retained, collectionPolicy{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/prompts/0/id" {
t.Fatalf("invalid retained records = %#v, %#v", retained, diagnostics)
}
}
func TestCollectionPolicyWithoutValueRejectsIdentityChange(t *testing.T) {
lower := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
current := collectionPolicy{identityFields: []string{"name"}, identityPresent: true}
retained, diagnostics := collectionPolicyWithoutValue(&lower, current, nil, Keys("prompts"))
if !reflect.DeepEqual(retained, collectionPolicy{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("no-value identity change = %#v, %#v", retained, diagnostics)
}
}
func TestCollectionPolicyLifetimePersistenceAndReplacementTable(t *testing.T) {
tests := []struct {
name string
current collectionPolicy
replaceValue bool
wantIdentity []string
wantIdentitySet bool
wantMode sequenceMode
wantRecords int
}{
{
name: "mode-only replace declaration does not reset retained identity",
current: collectionPolicy{
explicitMode: sequenceModeReplace, explicitModePresent: true,
},
wantIdentity: []string{"id"}, wantIdentitySet: true,
wantMode: sequenceModeReplace, wantRecords: 1,
},
{
name: "whole-sequence replacement resets lower identity",
replaceValue: true,
wantIdentitySet: false,
wantMode: sequenceModeAppend,
wantRecords: 0,
},
{
name: "replacement can establish a new identity",
current: collectionPolicy{identityFields: []string{"name"}, identityPresent: true},
replaceValue: true,
wantIdentity: []string{"name"},
wantIdentitySet: true,
wantMode: sequenceModeAppend,
wantRecords: 0,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
var got collectionPolicy
if test.replaceValue {
got = collectionPolicyForReplacement(&lower, test.current)
} else {
var diagnostics []Diagnostic
got, diagnostics = collectionPolicyWithoutValue(&lower, test.current, nil, Keys("prompts"))
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
}
if got.identityPresent != test.wantIdentitySet || !reflect.DeepEqual(got.identityFields, test.wantIdentity) || got.explicitMode != test.wantMode || len(got.recordOrder) != test.wantRecords {
t.Fatalf("transition result = %#v", got)
}
})
}
}
func TestCollectionPolicyValueReplacementResetsIdentityButRetainsExplicitMode(t *testing.T) {
lower := collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}
lower.setRecordPolicy(newCollectionRecordPolicy())
current := collectionPolicy{identityFields: []string{"name"}, identityPresent: true}
replaced := collectionPolicyForReplacement(&lower, current)
if !replaced.identityPresent || !reflect.DeepEqual(replaced.identityFields, []string{"name"}) {
t.Fatalf("replacement identity was not reset: %#v", replaced)
}
if replaced.explicitMode != sequenceModeAppend || !replaced.explicitModePresent || len(replaced.recordOrder) != 0 {
t.Fatalf("replacement failed to retain explicit mode or retire lower record policy: %#v", replaced)
}
cleared := collectionPolicyForReplacement(&lower, collectionPolicy{})
if cleared.identityPresent || len(cleared.identityFields) != 0 || cleared.explicitMode != sequenceModeAppend || !cleared.explicitModePresent {
t.Fatalf("replacement without a new key did not clear YAML identity while retaining mode: %#v", cleared)
}
}
func TestCollectionRecordRetentionAndReplacementUseOpaqueOwners(t *testing.T) {
oldRecord := newCollectionRecordPolicy()
oldRecord.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
retained := retainedCollectionRecordPolicy(oldRecord)
if retained.owner != oldRecord.owner {
t.Fatal("retained record did not keep its owner")
}
if got, ok := retained.scope.at(Keys("steps")); !ok || !got.identityPresent {
t.Fatalf("retained record lost its local policy: %#v, %t", got, ok)
}
if retained.scope.collections["/steps"].policy.identityFields[0] = "changed"; oldRecord.scope.collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("retained record policy aliases its source")
}
// A whole-record replacement at the same list position, even with equal
// authored identity/source metadata, receives new ownership and no old rules.
replacement := replacedCollectionRecordPolicy()
if replacement.owner == oldRecord.owner || len(replacement.scope.collections) != 0 {
t.Fatal("replacement record inherited discarded owner policy")
}
}
func TestCollectionPolicyAncestorRetirementPreservesIndependentMode(t *testing.T) {
var scope collectionPolicyScope
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
scope.set(Keys("catalog", "prompts"), collectionPolicyWithRecords(collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
}, owner))
scope.retireCollectionPoliciesAtOrBelow(Keys("catalog"))
policy, ok := scope.at(Keys("catalog", "prompts"))
if !ok || policy.identityPresent || len(policy.recordOrder) != 0 || policy.explicitMode != sequenceModeAppend || !policy.explicitModePresent {
t.Fatalf("ancestor retirement = %#v, %t", policy, ok)
}
reintroduced, diagnostics := collectionPolicyWithoutValue(&policy, collectionPolicy{}, nil, Keys("catalog", "prompts"))
if len(diagnostics) != 0 || reintroduced.identityPresent || reintroduced.explicitMode != sequenceModeAppend {
t.Fatalf("retired identity was resurrected while mode persisted: %#v, %#v", reintroduced, diagnostics)
}
}
func TestEqualOuterIdentitiesHaveDistinctRecordOwners(t *testing.T) {
retained := newCollectionRecordPolicy()
replacement := replacedCollectionRecordPolicy()
if retained.owner == replacement.owner {
t.Fatal("replacement with the same outer identity reused the old owner")
}
if retained.owner == nil || replacement.owner == nil {
t.Fatal("record owner token is missing")
}
}
func TestCollectionPolicyScopesKeepRootAndItemRulesIndependent(t *testing.T) {
var root collectionPolicyScope
root.set(Keys("steps"), collectionPolicy{identityFields: []string{"root-id"}, identityPresent: true})
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"item-id"}, identityPresent: true})
root.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, owner))
root.retireCollectionPoliciesAtOrBelow(Keys("steps"))
rootPolicy, ok := root.at(Keys("steps"))
if !ok || rootPolicy.identityPresent {
t.Fatalf("root-local policy was not retired: %#v", rootPolicy)
}
itemPolicy, ok := root.at(Keys("items"))
if !ok || len(itemPolicy.recordOrder) != 1 {
t.Fatalf("unrelated item owner disappeared: %#v", itemPolicy)
}
itemRecord, _ := itemPolicy.recordPolicyAt(0)
nested, ok := itemRecord.scope.at(Keys("steps"))
if !ok || !nested.identityPresent || !reflect.DeepEqual(nested.identityFields, []string{"item-id"}) {
t.Fatalf("item-local policy was coupled to root destination: %#v", nested)
}
}
func TestCollectionPolicyProjectionUsesConcretePlacement(t *testing.T) {
first := newCollectionRecordPolicy()
first.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"step"}, identityPresent: true})
second := newCollectionRecordPolicy()
second.scope.set(Keys("steps"), collectionPolicy{explicitMode: sequenceModeAppend, explicitModePresent: true})
var scope collectionPolicyScope
scope.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, first, second))
projected := projectCollectionPolicyScope(scope, Keys("mounted"))
if len(projected) != 3 {
t.Fatalf("projected policies = %#v", projected)
}
want := []string{"/mounted/items", "/mounted/items/0/steps", "/mounted/items/1/steps"}
for index, policy := range projected {
if policy.path.String() != want[index] {
t.Fatalf("projected path[%d] = %q, want %q", index, policy.path.String(), want[index])
}
}
}
func TestCollectionPolicyInheritanceCopiesOwnersAndState(t *testing.T) {
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
original := collectionPolicyWithRecords(collectionPolicy{}, owner)
copy := cloneCollectionPolicyForInheritance(original)
copyOwner := copy.recordOrder[0]
if copyOwner == owner.owner {
t.Fatal("inheritance copy retained the source record owner")
}
copy.recordPolicies[copyOwner].collections["/steps"].policy.identityFields[0] = "changed"
if owner.scope.collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("inheritance copy aliases source policy")
}
}
func TestAcquisitionScopeCloneOwnsCollectionPolicyCopy(t *testing.T) {
owner := newCollectionRecordPolicy()
owner.scope.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
var scope acquisitionScope
scope.collectionPolicies.set(Keys("items"), collectionPolicyWithRecords(collectionPolicy{}, owner))
cloned := cloneAcquisitionScope(scope)
clonedOwner := cloned.collectionPolicies.collections["/items"].policy.recordOrder[0]
cloned.collectionPolicies.collections["/items"].policy.recordPolicies[clonedOwner].collections["/steps"].policy.identityFields[0] = "changed"
sourceOwner := scope.collectionPolicies.collections["/items"].policy.recordOrder[0]
if scope.collectionPolicies.collections["/items"].policy.recordPolicies[sourceOwner].collections["/steps"].policy.identityFields[0] != "id" {
t.Fatal("acquisition scope clone aliases collection policy")
}
if clonedOwner != owner.owner {
t.Fatal("acquisition scope clone changed record ownership")
}
}
func collectionPolicyWithRecords(policy collectionPolicy, records ...collectionRecordPolicy) collectionPolicy {
for _, record := range records {
policy.setRecordPolicy(record)
}
return policy
}
func parsePolicyNode(t *testing.T, input string) *document.Node {
t.Helper()
parsed, diagnostics := parseSource(Bytes("collection-policy-state.yml", []byte(input)), "collection-policy-state", defaultParseLimits())
if len(diagnostics) != 0 {
t.Fatalf("parse policy value: %#v", diagnostics)
}
return parsed.document
}

View File

@@ -0,0 +1,192 @@
package engine
import (
"reflect"
"testing"
)
func TestResolveCollectionPolicyReconcilesDeclarationsBeforeComposition(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: [namespace, id]
mode: append
- destination: [prompts]
key: [id, namespace, id]
- destination: [prompts]
mode: append
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, nil)
if len(diagnostics) != 0 {
t.Fatalf("policy diagnostics = %#v", diagnostics)
}
if !resolved.declarationsPresent || !resolved.identityDeclarationPresent || !resolved.modeDeclarationPresent {
t.Fatalf("declaration presence was lost: %#v", resolved)
}
wantFields := []string{"id", "namespace"}
if !reflect.DeepEqual(resolved.policy.identityFields, wantFields) || !reflect.DeepEqual(resolved.effectiveIdentityFields, wantFields) {
t.Fatalf("effective identity = %#v / %#v, want %#v", resolved.policy.identityFields, resolved.effectiveIdentityFields, wantFields)
}
if !resolved.policy.identityPresent || !resolved.policy.explicitModePresent || resolved.policy.explicitMode != sequenceModeAppend || resolved.operation != sequenceModeAppend {
t.Fatalf("resolved explicit policy = %#v, operation %d", resolved.policy, resolved.operation)
}
if len(resolved.declarations) != 3 || resolved.declarations[0].identityLocation == nil || resolved.declarations[2].location.Line == 0 {
t.Fatalf("declaration locations were not retained: %#v", resolved.declarations)
}
}
func TestResolveCollectionPolicyRejectsConflictingPeersAtBothSites(t *testing.T) {
tests := []struct {
name string
body string
}{
{
name: "identity fields",
body: `configkit:
sequences:
- destination: [prompts]
key: id
- destination: [prompts]
key: name
`,
},
{
name: "explicit modes",
body: `configkit:
sequences:
- destination: [prompts]
mode: append
- destination: [prompts]
mode: replace
`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, test.body, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, nil)
if !reflect.DeepEqual(resolved, collectionPolicyResolution{}) || len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("resolution/diagnostics = %#v / %#v", resolved, diagnostics)
}
diagnostic := diagnostics[0]
if diagnostic.Path.String() != "/prompts" || diagnostic.Primary == nil || len(diagnostic.Related) != 1 {
t.Fatalf("conflict does not identify both declaration sites: %#v", diagnostic)
}
if diagnostic.Primary.SourceID != "sequence-declarations" || diagnostic.Related[0].SourceID != "sequence-declarations" || diagnostic.Primary.Line == diagnostic.Related[0].Line {
t.Fatalf("conflict locations = %#v and %#v", diagnostic.Primary, diagnostic.Related)
}
})
}
}
func TestResolveCollectionPolicySelectsCurrentThenRetainedExplicitMode(t *testing.T) {
lower := &collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
explicitMode: sequenceModeAppend,
explicitModePresent: true,
modeLocation: &Location{SourceID: "lower", Line: 2},
}
current := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
mode: upsert
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), current, nil, lower)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.explicitMode != sequenceModeUpsert || !resolved.policy.explicitModePresent {
t.Fatalf("current mode did not override retained mode: %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), nil, nil, lower)
if len(diagnostics) != 0 || resolved.operation != sequenceModeAppend || resolved.policy.explicitMode != sequenceModeAppend || !resolved.policy.explicitModePresent {
t.Fatalf("omitted current mode did not retain lower explicit mode: %#v, %#v", resolved, diagnostics)
}
if !reflect.DeepEqual(resolved.effectiveIdentityFields, []string{"id"}) {
t.Fatalf("retained identity = %#v", resolved.effectiveIdentityFields)
}
}
func TestResolveCollectionPolicyInfersWithoutPersistingMode(t *testing.T) {
keyOnly := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: id
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), keyOnly, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.explicitModePresent {
t.Fatalf("identity-based mode inference = %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), nil, []string{"id"}, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || resolved.policy.identityPresent || !resolved.applicationIdentityPresent {
t.Fatalf("application identity was not kept distinct from YAML policy: %#v, %#v", resolved, diagnostics)
}
noIdentity := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
`, rootDirectiveContextSource).sequences
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), noIdentity, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeReplace || resolved.policy.explicitModePresent {
t.Fatalf("unkeyed mode inference = %#v, %#v", resolved, diagnostics)
}
explicitDefault := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
mode: replace
`, rootDirectiveContextSource).sequences
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), explicitDefault, nil, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeReplace || !resolved.policy.explicitModePresent {
t.Fatalf("explicit default mode lost its presence: %#v, %#v", resolved, diagnostics)
}
}
func TestResolveCollectionPolicyChecksKeyOnlyAgainstApplicationConstraint(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: [namespace, id]
`, rootDirectiveContextSource).sequences
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, []string{"id", "namespace", "id"}, nil)
if len(diagnostics) != 0 || resolved.operation != sequenceModeUpsert || !reflect.DeepEqual(resolved.effectiveIdentityFields, []string{"id", "namespace"}) {
t.Fatalf("equivalent key-only declaration was rejected: %#v, %#v", resolved, diagnostics)
}
resolved, diagnostics = resolveCollectionPolicy(Keys("prompts"), declarations, []string{"slug"}, nil)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict {
t.Fatalf("conflicting key-only declaration = %#v, %#v", resolved, diagnostics)
}
if diagnostics[0].Primary == nil || diagnostics[0].Path.String() != "/prompts" || len(diagnostics[0].Related) != 0 {
t.Fatalf("application conflict diagnostic = %#v", diagnostics[0])
}
}
func TestResolveCollectionPolicyChecksRetainedIdentityAgainstApplicationConstraint(t *testing.T) {
lower := &collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), nil, []string{"name"}, lower)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/prompts" {
t.Fatalf("retained/application conflict = %#v, %#v", resolved, diagnostics)
}
}
func TestResolveCollectionPolicyRequiresCurrentIdentityToAgreeWithRetainedIdentity(t *testing.T) {
declarations := extractSequenceDeclarationsYAML(t, `configkit:
sequences:
- destination: [prompts]
key: name
`, rootDirectiveContextSource).sequences
lower := &collectionPolicy{
identityFields: []string{"id"},
identityPresent: true,
identityLocation: &Location{SourceID: "lower", Line: 4},
}
resolved, diagnostics := resolveCollectionPolicy(Keys("prompts"), declarations, nil, lower)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/prompts" {
t.Fatalf("current/retained identity conflict = %#v, %#v", resolved, diagnostics)
}
if diagnostics[0].Primary == nil || len(diagnostics[0].Related) != 1 || diagnostics[0].Related[0].SourceID != "lower" {
t.Fatalf("current/retained conflict locations = %#v", diagnostics[0])
}
}

View File

@@ -0,0 +1,81 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
// A true entry replaces the collection itself as well as its descendants;
// false records ordinary ancestor replacement without selecting a collection
// operation at that exact path.
type collectionReplacementPaths map[string]bool
func (paths collectionReplacementPaths) contains(path Path, includeExact bool) bool {
for length := 0; length <= len(path.segments); length++ {
prefix := Path{segments: path.segments[:length]}
inclusive, found := paths[prefix.String()]
if found && (length < len(path.segments) || includeExact && inclusive) {
return true
}
}
return false
}
// lowerCollectionPoliciesForClaims makes replacement eligibility independent of
// declaration order. It removes only old YAML identity/record state; new pending
// declarations and application constraints are reconciled afterward.
func lowerCollectionPoliciesForClaims(
lower collectionPolicyScope,
claims []compositionClaim,
declarations map[string][]sequenceDeclarationSpec,
application sequencePolicy,
) (collectionPolicyScope, collectionReplacementPaths) {
byPath := make(map[string]compositionClaim, len(claims))
rootPath := ""
if len(claims) != 0 {
rootPath = claims[0].path.String()
}
for _, claim := range claims {
byPath[claim.path.String()] = claim
}
replaced := make(collectionReplacementPaths)
for key, claim := range byPath {
old, hasOld := lower.collections[key]
current := declarations[key]
if claim.kind == document.Mapping && (key == rootPath || !hasOld && len(current) == 0) {
continue // Ordinary mapping combination preserves descendants.
}
mode := sequenceMode(0)
identity := old.policy.identityPresent
for _, declaration := range current {
identity = identity || declaration.identityPresent
if mode == 0 && declaration.modePresent {
mode = declaration.mode
}
}
retainedMode := sequenceMode(0)
if old.policy.explicitModePresent {
retainedMode = old.policy.explicitMode
}
_, registered := application.identityFieldsAt(claim.path)
mode = collectionOperationMode(mode, retainedMode, identity || registered)
if mode == sequenceModeReplace {
replaced[key] = true
} else if claim.kind != document.Mapping && claim.kind != document.Sequence {
replaced[key] = false
}
}
result := cloneCollectionPolicyScope(lower)
for key, entry := range result.collections {
if replaced.contains(entry.destination, true) {
clearCollectionIdentity(&entry.policy)
result.collections[key] = entry
}
}
return result, replaced
}
func clearCollectionIdentity(policy *collectionPolicy) {
policy.identityFields = nil
policy.identityPresent = false
policy.identityLocation = nil
policy.recordOrder = nil
policy.recordPolicies = nil
}

View File

@@ -0,0 +1,111 @@
package engine
import (
"errors"
"testing"
)
func TestSourceReplacementRetiresIdentityForEveryValueKind(t *testing.T) {
t.Parallel()
for _, value := range []string{"null", "42", "{}", "[{name: new}]"} {
t.Run(value, func(t *testing.T) {
lower := "configkit:\n sequences: [{destination: [items], key: id}]\nitems: [{id: old}]\n"
main := "configkit:\n sequences: [{destination: [items], key: name, mode: replace}]\nitems: " + value + "\n"
loaded, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(lower))}}}},
})
if err != nil || loaded == nil {
t.Fatalf("replacement retained obsolete identity: %v", err)
}
})
}
}
func TestAncestorRemovalCanEstablishPendingReplacementIdentity(t *testing.T) {
t.Parallel()
for _, inheritance := range []bool{false, true} {
name := "tiers"
if inheritance {
name = "inheritance"
}
t.Run(name, func(t *testing.T) {
var loaded *testResult[map[string]any]
var err error
if inheritance {
loaded, err = testLoad[map[string]any](Bytes("source", []byte(`configkit:
sequences:
- {destination: [base, group, items], key: id}
- {destination: [child, group, items], key: name}
base: {group: {items: [{id: old}]}}
child:
extends: base
group: null
grandchild:
extends: child
group: {items: [{name: new}]}
`)))
} else {
loaded, err = testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte("group: {items: [{name: new}]}\n"))},
underlays: []loadGroup{
{inputs: []loadInput{{source: Bytes("base", []byte("configkit:\n sequences: [{destination: [group, items], key: id}]\ngroup: {items: [{id: old}]}\n"))}}},
{inputs: []loadInput{{source: Bytes("remove", []byte("configkit:\n sequences: [{destination: [group, items], key: name}]\ngroup: null\n"))}}},
},
})
}
if err != nil {
t.Fatal(err)
}
value := loaded.Config
if inheritance {
value = value["grandchild"].(map[string]any)
}
items := value["group"].(map[string]any)["items"].([]any)
if len(items) != 1 || items[0].(map[string]any)["name"] != "new" {
t.Fatalf("reintroduced collection = %#v", items)
}
})
}
}
func TestInheritanceAncestorRetirementDoesNotResurrectOldIdentity(t *testing.T) {
t.Parallel()
_, err := testLoad[map[string]any](Bytes("source", []byte(`configkit:
sequences: [{destination: [base, group, items], key: id}]
base: {group: {items: [{id: old}]}}
child:
extends: base
group: null
grandchild:
extends: child
group: {items: [{name: new}]}
`)))
if err != nil {
t.Fatal(err)
}
}
func TestReplacementRetirementPreservesApplicationConstraintsAndRetainedKeys(t *testing.T) {
t.Parallel()
for _, withValue := range []bool{false, true} {
main := "configkit:\n sequences: [{destination: [items], key: name, mode: replace}]\n"
if withValue {
main += "items: null\n"
}
definition := Definition[map[string]any]{}
if withValue {
definition.sequenceIdentityRegistrations = []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"},
}}
}
_, err := testGroupedLoad(loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte(main))}, definition: definition,
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences: [{destination: [items], key: id}]\nitems: [{id: old}]\n"))}}}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != CodeSequenceConflict {
t.Fatalf("retirement bypassed identity agreement: %v", err)
}
}
}

View File

@@ -1,21 +1,26 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
import (
"maps"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
// composedDocument is the private handoff between source assembly and later
// precedence layers. Claims remain ordered by source and declaration traversal;
// origins retain every source that contributes to a composite mapping.
type composedDocument struct {
root *document.Node
acquisition []acquisitionScope
importGraph importInclusionGraph
claims []compositionClaim
sequenceRanges []sequenceRange
origins map[string]Origin
overlays []overlayRecord
applications []historyApplication
inputBytes int64
inputNodes int
root *document.Node
acquisition []acquisitionScope
importGraph importInclusionGraph
collectionPolicies collectionPolicyScope
claims []compositionClaim
sequenceRanges []sequenceRange
origins map[string]Origin
overlays []overlayRecord
applications []historyApplication
inputBytes int64
inputNodes int
}
type sequenceHistoryMode uint8
@@ -23,13 +28,16 @@ type sequenceHistoryMode uint8
const (
sequenceHistoryReplace sequenceHistoryMode = iota + 1
sequenceHistoryAppend
sequenceHistoryMapped
)
type sequenceHistoryContribution struct {
path Path
mode sequenceHistoryMode
offset int
length int
path Path
mode sequenceHistoryMode
offset int
length int
items []sequenceItemProjection
replacements []sequenceItemReplacement
}
// assemblyAccumulator owns one mutable working document. Acquired trees held by
@@ -51,10 +59,11 @@ func cloneComposedDocument(base *composedDocument) *composedDocument {
}
return &composedDocument{
root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims),
acquisition: cloneAcquisitionScopes(base.acquisition),
importGraph: base.importGraph.clone(),
sequenceRanges: cloneSequenceRanges(base.sequenceRanges),
origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays),
acquisition: cloneAcquisitionScopes(base.acquisition),
importGraph: base.importGraph.clone(),
collectionPolicies: cloneCollectionPolicyScope(base.collectionPolicies),
sequenceRanges: cloneSequenceRanges(base.sequenceRanges),
origins: cloneOriginIndex(base.origins), overlays: cloneOverlayRecords(base.overlays),
applications: cloneHistoryApplications(base.applications),
inputBytes: base.inputBytes, inputNodes: base.inputNodes,
}
@@ -80,6 +89,53 @@ type sequenceRange struct {
path Path
start int
end int
items []sequenceItemProjection
projection sequenceItemProjectionIndex
}
func (assigned sequenceRange) outputForInput(inputIndex int) (int, bool) {
if assigned.projection.outputByInput != nil {
return assigned.projection.outputForInput(inputIndex)
}
if assigned.items != nil {
output, found := 0, false
for _, item := range assigned.items {
if item.inputIndex == inputIndex && inputIndex >= 0 && item.outputIndex >= 0 {
output, found = item.outputIndex, true
}
}
return output, found
}
if inputIndex < 0 || inputIndex >= assigned.end-assigned.start {
return 0, false
}
return assigned.start + inputIndex, true
}
func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) {
if assigned.projection.inputByOutput != nil {
return assigned.projection.inputForOutput(outputIndex)
}
if assigned.items != nil {
input, found := 0, false
for _, item := range assigned.items {
if item.outputIndex == outputIndex && outputIndex >= 0 && item.inputIndex >= 0 {
input, found = item.inputIndex, true
}
}
return input, found
}
if outputIndex < assigned.start || outputIndex >= assigned.end {
return 0, false
}
return outputIndex - assigned.start, true
}
func indexSequenceRange(assigned sequenceRange) sequenceRange {
if assigned.items != nil && assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil {
assigned.projection = newSequenceItemProjectionIndex(assigned.items)
}
return assigned
}
// composeDocuments recursively joins disjoint mapping entries. Source order
@@ -93,6 +149,17 @@ func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic
// sequence rules when policy contains registrations. The ordinary adapter above
// deliberately preserves the existing unregistered behavior for current callers.
func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) (*composedDocument, []Diagnostic) {
return composeDocumentsWithCollectionPolicy(policy, collectionPolicyScope{}, nil, inputs...)
}
// composeDocumentsWithCollectionPolicy resolves collection declarations and
// eligible lower policy before strictly composing completed peer documents.
func composeDocumentsWithCollectionPolicy(
policy sequencePolicy,
lower collectionPolicyScope,
declarations []sequenceDeclarationSpec,
inputs ...*document.Node,
) (*composedDocument, []Diagnostic) {
if len(inputs) == 0 {
return nil, []Diagnostic{compositionInvalidDiagnostic("strict composition requires at least one document", nil)}
}
@@ -110,6 +177,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
collected.append(input, Path{}, sourceIndex)
}
identityFieldsByPath, collectionPolicies, diagnostics := resolveStrictCollectionPolicies(
collected.ordered, policy, lower, declarations,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
checked := make(map[string]struct{}, len(collected.byPath))
for _, claim := range collected.ordered {
pointer := claim.path.String()
@@ -118,13 +192,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
}
checked[pointer] = struct{}{}
claims := collected.byPath[pointer]
if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && policy.registered(claim.path)) {
if len(claims) < 2 || allMappingClaims(claims) || (allSequenceClaims(claims) && len(identityFieldsByPath[pointer]) != 0) {
continue
}
return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)}
}
validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy)
validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, identityFieldsByPath)
if len(diagnostics) != 0 {
return nil, diagnostics
}
@@ -132,7 +206,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
root := document.Clone(inputs[0])
index := new(mappingIndex)
for sourceIndex, input := range inputs[1:] {
mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, policy, validated)
mergeComposedMappings(root, input, Path{}, index, sourceIndex+1, identityFieldsByPath, validated)
}
applications := make([]historyApplication, 0, len(inputs))
seenSequences := make(map[string]bool)
@@ -159,11 +233,12 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
origins := projectedCompositionOrigins(inputs, ranges)
completeCompositionOrigins(root, Path{}, origins)
return &composedDocument{
root: root,
claims: cloneCompositionClaims(collected.ordered),
sequenceRanges: cloneSequenceRanges(ranges),
origins: origins,
applications: applications,
root: root,
collectionPolicies: collectionPolicies,
claims: cloneCompositionClaims(collected.ordered),
sequenceRanges: cloneSequenceRanges(ranges),
origins: origins,
applications: applications,
}, nil
}
@@ -219,71 +294,128 @@ type validatedSequence struct {
items []*document.Node
}
func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
func resolveStrictCollectionPolicies(
claims []compositionClaim,
applicationPolicy sequencePolicy,
lower collectionPolicyScope,
declarations []sequenceDeclarationSpec,
) (map[string][]string, collectionPolicyScope, []Diagnostic) {
paths := make([]Path, 0, len(claims)+len(declarations))
seen := make(map[string]bool, len(claims)+len(declarations))
valuePresent := make(map[string]bool)
declarationsByPath := make(map[string][]sequenceDeclarationSpec, len(declarations))
for _, claim := range claims {
if len(lower.collections) != 0 {
valuePresent[claim.path.String()] = true
}
if claim.kind != document.Sequence {
continue
}
key := claim.path.String()
if !seen[key] {
seen[key] = true
paths = append(paths, claim.path.clone())
}
}
for _, declaration := range declarations {
key := declaration.destination.String()
declarationsByPath[key] = append(declarationsByPath[key], declaration)
if !seen[key] {
seen[key] = true
paths = append(paths, declaration.destination.clone())
}
}
identityFieldsByPath := make(map[string][]string)
effectiveLower := lower
if len(lower.collections) != 0 {
effectiveLower, _ = lowerCollectionPoliciesForClaims(lower, claims, declarationsByPath, applicationPolicy)
}
collectionPolicies := cloneCollectionPolicyScope(effectiveLower)
var diagnostics []Diagnostic
for _, path := range paths {
key := path.String()
currentDeclarations := declarationsByPath[key]
lowerEntry, hasLower := effectiveLower.collections[key]
var lowerPolicy *collectionPolicy
if hasLower {
copy := cloneCollectionPolicy(lowerEntry.policy)
lowerPolicy = &copy
}
applicationIdentity, _ := applicationPolicy.identityFieldsAt(path)
resolution, found := resolveCollectionPolicy(path, currentDeclarations, applicationIdentity, lowerPolicy)
if len(found) != 0 {
diagnostics = append(diagnostics, found...)
continue
}
if len(resolution.effectiveIdentityFields) != 0 {
identityFieldsByPath[key] = cloneStrings(resolution.effectiveIdentityFields)
}
policyForPath := resolution.policy
if hasLower && !valuePresent[key] {
// A declaration without a new value changes policy metadata but
// leaves retained record ownership untouched.
policyForPath = cloneCollectionPolicy(lowerEntry.policy)
policyForPath.identityFields = cloneStrings(resolution.policy.identityFields)
policyForPath.identityPresent = resolution.policy.identityPresent
policyForPath.identityLocation = cloneLocation(resolution.policy.identityLocation)
policyForPath.explicitMode = resolution.policy.explicitMode
policyForPath.explicitModePresent = resolution.policy.explicitModePresent
policyForPath.modeLocation = cloneLocation(resolution.policy.modeLocation)
}
if collectionPolicies.collections == nil {
collectionPolicies.collections = make(map[string]collectionPolicyEntry)
}
collectionPolicies.collections[key] = collectionPolicyEntry{destination: path.clone(), policy: policyForPath}
}
return identityFieldsByPath, collectionPolicies, canonicalDiagnostics(diagnostics)
}
func validateRegisteredSequences(claims []compositionClaim, identityFieldsByPath map[string][]string) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
itemsByPath := make(map[string]map[int]validatedSequence)
identitiesByPath := make(map[string]map[string]Location)
identitiesByPath := make(map[string]map[string]document.Location)
lengthsByPath := make(map[string]int)
var ranges []sequenceRange
for _, claim := range claims {
if claim.kind != document.Sequence {
continue
}
identityField, registered := policy.identityFields[claim.path.String()]
if !registered {
identityFields := identityFieldsByPath[claim.path.String()]
if len(identityFields) == 0 {
continue
}
node := claim.node
pathKey := claim.path.String()
identities := identitiesByPath[pathKey]
if identities == nil {
identities = make(map[string]Location, len(node.Sequence))
identities = make(map[string]document.Location, len(node.Sequence))
identitiesByPath[pathKey] = identities
}
start := lengthsByPath[pathKey]
contribution := validatedSequence{items: node.Sequence}
for itemIndex, item := range node.Sequence {
itemPath := claim.path.Index(start + itemIndex)
if item == nil || item.Kind != document.Mapping {
location := (*Location)(nil)
if item != nil {
converted := publicLocation(item.Location)
location = &converted
}
return nil, nil, []Diagnostic{{
Code: CodeComposeInvalid, Severity: SeverityError,
Path: itemPath, Message: "registered sequence items must be mappings", Primary: location,
}}
}
fieldIndex := documentEntryIndex(item.Mapping, identityField)
if fieldIndex < 0 {
location := publicLocation(item.Location)
return nil, nil, []Diagnostic{{
Code: CodeComposeInvalid, Severity: SeverityError,
Path: itemPath, Message: "registered sequence item is missing its identity field",
Primary: &location,
}}
}
field := item.Mapping[fieldIndex].Value
if field == nil || field.Kind != document.String {
location := publicLocation(item.Mapping[fieldIndex].ValueLocation)
if field != nil {
location = publicLocation(field.Location)
}
return nil, nil, []Diagnostic{{
Code: CodeComposeInvalid, Severity: SeverityError,
Path: itemPath.Key(identityField), Message: "registered sequence identity must be a string",
Primary: &location,
}}
}
if first, duplicate := identities[field.Text]; duplicate {
current := publicLocation(field.Location)
records, diagnostics := validateNormalizedSequenceIdentityFrom(node, claim.path, identityFields, sequenceIdentityDiagnosticCodes{
invalid: CodeComposeInvalid, conflict: CodeComposeConflict,
}, start, sequenceStructuralRecord)
if len(diagnostics) != 0 {
return nil, nil, diagnostics
}
byIndex := make([]sequenceIdentityRecord, len(node.Sequence))
for _, record := range records {
byIndex[record.index] = record
}
for _, record := range byIndex {
if first, duplicate := identities[record.tuple]; duplicate {
current := publicLocation(record.location)
return nil, nil, []Diagnostic{{
Code: CodeComposeConflict, Severity: SeverityError,
Path: claim.path.clone(), Message: "registered sequence contains duplicate identities",
Primary: &first, Related: []Location{current},
Primary: ptrLocation(publicLocation(first)), Related: []Location{current},
}}
}
identities[field.Text] = publicLocation(field.Location)
identities[record.tuple] = record.location
}
if itemsByPath[pathKey] == nil {
itemsByPath[pathKey] = make(map[int]validatedSequence)
@@ -297,7 +429,7 @@ func validateRegisteredSequences(claims []compositionClaim, policy sequencePolic
return itemsByPath, ranges, nil
}
func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) {
func mergeComposedMappings(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
for _, incomingEntry := range incoming.Mapping {
entryPath := path.Key(incomingEntry.Key)
position := index.find(target, incomingEntry.Key)
@@ -308,7 +440,7 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
continue
}
current := target.Mapping[position].Value
if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && policy.registered(entryPath) {
if current.Kind == document.Sequence && incomingEntry.Value.Kind == document.Sequence && len(identityFieldsByPath[entryPath.String()]) != 0 {
contribution := validated[entryPath.String()][sourceIndex]
if contribution.items != nil {
for _, item := range contribution.items {
@@ -317,20 +449,15 @@ func mergeComposedMappings(target, incoming *document.Node, path Path, index *ma
}
continue
}
mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, policy, validated)
mergeDisjointMappingsAtPath(current, incomingEntry.Value, entryPath, index, sourceIndex, identityFieldsByPath, validated)
}
}
func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, policy sequencePolicy, validated map[string]map[int]validatedSequence) {
func mergeDisjointMappingsAtPath(target, incoming *document.Node, path Path, index *mappingIndex, sourceIndex int, identityFieldsByPath map[string][]string, validated map[string]map[int]validatedSequence) {
if target.Kind != document.Mapping || incoming.Kind != document.Mapping {
return
}
mergeComposedMappings(target, incoming, path, index, sourceIndex, policy, validated)
}
func (p sequencePolicy) registered(path Path) bool {
_, ok := p.identityFields[path.String()]
return ok
mergeComposedMappings(target, incoming, path, index, sourceIndex, identityFieldsByPath, validated)
}
func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {
@@ -338,6 +465,11 @@ func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {
for i, assigned := range ranges {
cloned[i] = assigned
cloned[i].path = assigned.path.clone()
if assigned.items != nil {
cloned[i].items = append([]sequenceItemProjection{}, assigned.items...)
}
cloned[i].projection.outputByInput = maps.Clone(assigned.projection.outputByInput)
cloned[i].projection.inputByOutput = maps.Clone(assigned.projection.inputByOutput)
}
return cloned
}
@@ -377,12 +509,28 @@ func documentEntryIndex(entries []document.Entry, key string) int {
func projectedCompositionOrigins(inputs []*document.Node, ranges []sequenceRange) map[string]Origin {
origins := make(map[string]Origin)
rangeBySourcePath := make(map[sequenceRangeKey]int, len(ranges))
for index, assigned := range ranges {
rangeBySourcePath[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = index
sequencesBySource := make([][]sequenceHistoryContribution, len(inputs))
for _, assigned := range ranges {
contribution := sequenceHistoryContribution{
path: assigned.path.clone(), mode: sequenceHistoryAppend,
offset: assigned.start, length: assigned.end - assigned.start,
}
if assigned.items != nil {
contribution.mode = sequenceHistoryMapped
contribution.items = assigned.items
}
sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], contribution)
}
for sourceIndex, input := range inputs {
collectProjectedCompositionOrigins(input, Path{}, Path{}, sourceIndex, ranges, rangeBySourcePath, origins)
application := historyApplication{value: input, sequences: sequencesBySource[sourceIndex]}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
walkHistoryApplication(application, func(_, effective Path, node *document.Node) {
pointer := effective.String()
origin := origins[pointer]
origin.Layer = LayerBase
origin.Locations = append(origin.Locations, publicLocation(node.Location))
origins[pointer] = origin
})
}
return origins
}
@@ -400,39 +548,12 @@ func indexSequenceRanges(ranges []sequenceRange) sequenceRangeIndex {
}
index := make(sequenceRangeIndex, len(ranges))
for _, assigned := range ranges {
assigned = indexSequenceRange(assigned)
index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned
}
return index
}
// Keep local paths for matching Stage 2 ranges and effective paths for emitted
// evidence. The paths are transformed segment by segment; pointer strings are
// only used as map keys after projection is complete.
func collectProjectedCompositionOrigins(node *document.Node, localPath, effectivePath Path, sourceIndex int, ranges []sequenceRange, rangeBySourcePath map[sequenceRangeKey]int, origins map[string]Origin) {
pointer := effectivePath.String()
origin := origins[pointer]
origin.Layer = LayerBase
origin.Locations = append(origin.Locations, publicLocation(node.Location))
origins[pointer] = origin
switch node.Kind {
case document.Mapping:
for _, entry := range node.Mapping {
collectProjectedCompositionOrigins(entry.Value, localPath.Key(entry.Key), effectivePath.Key(entry.Key), sourceIndex, ranges, rangeBySourcePath, origins)
}
case document.Sequence:
start := 0
if rangeIndex, ok := rangeBySourcePath[sequenceRangeKey{sourceIndex: sourceIndex, path: localPath.String()}]; ok {
start = ranges[rangeIndex].start
}
for index, child := range node.Sequence {
localChild := localPath.Index(index)
effectiveChild := effectivePath.Index(start + index)
collectProjectedCompositionOrigins(child, localChild, effectiveChild, sourceIndex, ranges, rangeBySourcePath, origins)
}
}
}
func applyCompositionOrigins(decoded *decodedConfiguration, origins map[string]Origin) {
for pointer, origin := range origins {
if _, present := decoded.presence[pointer]; present {

View File

@@ -10,17 +10,20 @@ import (
)
func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) {
policy := sequencePolicy{identityFields: make(map[string]string)}
registrations := make([]sequenceIdentityRegistration, 0, 128)
var inputs []*document.Node
for source := 0; source < 3; source++ {
var text strings.Builder
for collection := 0; collection < 128; collection++ {
key := fmt.Sprintf("records%d", collection)
policy.identityFields[Keys(key).String()] = "id"
registrations = append(registrations, sequenceIdentityRegistration{
selector: sequenceIdentitySelector{literalSequenceSelector(key)}, identityFields: []string{"id"},
})
fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source)
}
inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String()))
}
policy := mustCompileSequenceIdentityPolicy(t, registrations)
composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
@@ -242,7 +245,7 @@ func TestComposeDocumentsAppendsRegisteredSequencesAndReturnsRanges(t *testing.T
first := parseCompositionDocument(t, "source-1", "records: [{id: '', left: 1}, {id: ' spaced ', right: 1}]\nother: [one]\n")
second := parseCompositionDocument(t, "source-2", "records: [{id: ' ', text: second}]\n")
firstBefore, secondBefore := document.Clone(first), document.Clone(second)
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
if len(diagnostics) != 0 {
@@ -287,10 +290,10 @@ escaped/~:
name: escaped-second-name
`)
firstBefore, secondBefore := document.Clone(first), document.Clone(second)
policy := sequencePolicy{identityFields: map[string]string{
Keys("mounted", "records").String(): "id",
Keys("escaped/~", "7").String(): "id",
}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("mounted"), literalSequenceSelector("records")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("escaped/~"), literalSequenceSelector("7")}, identityFields: []string{"id"}},
})
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
if len(diagnostics) != 0 {
@@ -357,6 +360,78 @@ escaped/~:
}
}
func TestProjectedCompositionOriginsUseNoncontiguousItemMappings(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
label: equal
- id: middle
- id: last
nested: {old: lower}
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
nested: {new: higher}
- id: first
label: equal
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher, sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
inputs := []*document.Node{lowerRoot, higherRoot}
ranges := []sequenceRange{
{
sourceIndex: 0, path: Keys("records"), start: 0, end: 3,
items: combined.lowerItems,
}, {
sourceIndex: 1, path: Keys("records"), start: 0, end: 3,
items: combined.higherItems,
},
}
origins := projectedCompositionOrigins(inputs, ranges)
if got, want := sequenceIdentityValues(combined.value, "id"), []string{"first", "middle", "last", "added"}; !reflect.DeepEqual(got, want) {
t.Fatalf("combined IDs = %#v, want %#v", got, want)
}
for path, want := range map[string][]string{
"/records/0/label": {"higher"},
"/records/1/id": {"lower"},
"/records/2/nested/new": {"higher"},
"/records/3/id": {"higher"},
"/records/0/nested/new": nil,
"/records/2/nested/old": nil,
} {
origin := origins[path]
var got []string
for _, location := range origin.Locations {
got = append(got, location.SourceID)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("origin sources at %s = %#v, want %#v", path, got, want)
}
}
}
func TestCloneSequenceRangesPreservesPreparedProjection(t *testing.T) {
ranges := []sequenceRange{{
start: 0, end: 1,
projection: newSequenceItemProjectionIndex([]sequenceItemProjection{{inputIndex: 0, outputIndex: 4}}),
}}
cloned := cloneSequenceRanges(ranges)
if output, ok := cloned[0].outputForInput(0); !ok || output != 4 {
t.Fatalf("cloned projection = %d, %v; want output index 4", output, ok)
}
cloned[0].projection.outputByInput[0] = 9
if output, ok := ranges[0].outputForInput(0); !ok || output != 4 {
t.Fatalf("cloned projection aliases original: %d, %v", output, ok)
}
}
func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
tests := []struct {
name string
@@ -365,15 +440,12 @@ func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
wantCode Code
}{
{name: "nonmapping item", body: "records: [plain]\n", path: "/records/0", wantCode: CodeComposeInvalid},
{name: "missing identity", body: "records: [{name: plain}]\n", path: "/records/0", wantCode: CodeComposeInvalid},
{name: "null identity", body: "records: [{id: null}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "numeric identity", body: "records: [{id: 12}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "boolean identity", body: "records: [{id: false}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
{name: "missing identity", body: "records: [{name: plain}]\n", path: "/records/0/id", wantCode: CodeComposeInvalid},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}},
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", test.body),
)
if len(diagnostics) != 1 || diagnostics[0].Code != test.wantCode || diagnostics[0].Path.String() != test.path {
@@ -386,14 +458,138 @@ func TestComposeDocumentsValidatesRegisteredSequenceSingletons(t *testing.T) {
}
}
func TestComposeDocumentsUsesLowerIdentityForStrictPeers(t *testing.T) {
path := Keys("records")
lower := collectionPolicyScope{collections: map[string]collectionPolicyEntry{
path.String(): {
destination: path,
policy: collectionPolicy{identityFields: []string{"id"}, identityPresent: true},
},
}}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, nil,
parseCompositionDocument(t, "source-1", "records: [{id: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("compose strict peers with lower identity: %#v", diagnostics)
}
value, present := documentAtPath(composed.root, path)
if !present || value.Kind != document.Sequence || len(value.Sequence) != 2 {
t.Fatalf("composed records = %#v, %v; want two strict peer records", value, present)
}
resolved, present := composed.collectionPolicies.collections[path.String()]
if !present || !resolved.policy.identityPresent || !reflect.DeepEqual(resolved.policy.identityFields, []string{"id"}) {
t.Fatalf("composed collection policy = %#v, %v; want retained lower identity", resolved, present)
}
clone := cloneComposedDocument(composed)
resolved.policy.identityFields[0] = "mutated"
if !reflect.DeepEqual(clone.collectionPolicies.collections[path.String()].policy.identityFields, []string{"id"}) {
t.Fatal("cloned composition shares mutable collection identity fields")
}
}
func TestComposeDocumentsResolvesDeclarationsBeforePeerComposition(t *testing.T) {
path := Keys("records")
declaration := func(source string, line int, fields ...string) sequenceDeclarationSpec {
location := Location{SourceID: source, Line: line}
return sequenceDeclarationSpec{
destination: path, identityFields: fields, identityPresent: true,
identityLocation: &location,
}
}
_, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "id"), declaration("source-2", 4, "name")},
parseCompositionDocument(t, "source-1", "left: true\n"),
parseCompositionDocument(t, "source-2", "right: true\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("conflicting policy-only declarations = %#v", diagnostics)
}
if diagnostics[0].Primary == nil || len(diagnostics[0].Related) != 1 {
t.Fatalf("declaration conflict lacks both locations: %#v", diagnostics[0])
}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "tenant", "id"), declaration("source-2", 4, "id", "tenant")},
parseCompositionDocument(t, "source-1", "records: [{tenant: true, id: null}, {tenant: \"alpha\", id: 1}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: 1.0, tenant: \"beta\"}, {id: \"1\", tenant: \"alpha\"}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("compose normalized compound identities: %#v", diagnostics)
}
value, present := documentAtPath(composed.root, path)
if !present || len(value.Sequence) != 4 {
t.Fatalf("compound identity records = %#v, %v; want four distinct records", value, present)
}
_, diagnostics = composeDocumentsWithCollectionPolicy(
sequencePolicy{}, collectionPolicyScope{},
[]sequenceDeclarationSpec{declaration("source-1", 2, "id")},
parseCompositionDocument(t, "source-1", "records: [{id: 1}]\n"),
parseCompositionDocument(t, "source-2", "records: [{id: 1.0}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("equivalent numeric peer identities = %#v", diagnostics)
}
}
func TestComposeDocumentsReplacementDoesNotBorrowLowerYAMLIdentity(t *testing.T) {
path := Keys("records")
lower := collectionPolicyScope{collections: map[string]collectionPolicyEntry{
path.String(): {
destination: path,
policy: collectionPolicy{identityFields: []string{"id"}, identityPresent: true},
},
}}
declaration := sequenceDeclarationSpec{destination: path, mode: sequenceModeReplace, modePresent: true}
_, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, []sequenceDeclarationSpec{declaration},
parseCompositionDocument(t, "source-1", "records: [{name: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: second}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("replacement peers borrowing lower identity = %#v", diagnostics)
}
identityReplacement := sequenceDeclarationSpec{
destination: path, identityFields: []string{"name"}, identityPresent: true,
mode: sequenceModeReplace, modePresent: true,
}
composed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, lower, []sequenceDeclarationSpec{identityReplacement},
parseCompositionDocument(t, "source-1", "records: [{name: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatalf("replacement with a new identity declaration: %#v", diagnostics)
}
resolved := composed.collectionPolicies.collections[path.String()].policy
if !reflect.DeepEqual(resolved.identityFields, []string{"name"}) || !resolved.identityPresent {
t.Fatalf("replacement collection identity = %#v; want name", resolved)
}
_, diagnostics = composeDocumentsWithCollectionPolicy(
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}), lower,
[]sequenceDeclarationSpec{declaration},
parseCompositionDocument(t, "source-1", "records: [{name: replacement}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/0/id" {
t.Fatalf("replacement ignored application identity constraint = %#v", diagnostics)
}
}
func TestComposeDocumentsReportsAssignedPathForMalformedRegisteredSequenceContribution(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}},
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "records: [{id: first}]\n"),
parseCompositionDocument(t, "source-2", "records: [{name: malformed}]\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/1" {
t.Fatalf("diagnostics = %#v, want invalid item at /records/1", diagnostics)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/records/1/id" {
t.Fatalf("diagnostics = %#v, want invalid identity field at /records/1/id", diagnostics)
}
if diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "source-2" {
t.Fatalf("malformed item location = %#v, want source-2", diagnostics[0].Primary)
@@ -420,7 +616,7 @@ func TestComposeDocumentsRejectsRegisteredDuplicateIdentities(t *testing.T) {
inputs = append(inputs, parseCompositionDocument(t, "source-2", test.second))
}
_, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}}, inputs...,
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}), inputs...,
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/records" {
t.Fatalf("diagnostics = %#v, want duplicate conflict at /records", diagnostics)
@@ -438,9 +634,12 @@ func TestComposeDocumentsRejectsRegisteredDuplicateIdentities(t *testing.T) {
}
func TestComposeDocumentsSkipsMissingAndNonSequenceRegistrations(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{
"/absent": "id", "/not-a-sequence": "id", "/catalog": "id", "/catalog/records": "id",
}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("absent")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("not-a-sequence")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("catalog")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("catalog"), literalSequenceSelector("records")}, identityFields: []string{"id"}},
})
composed, diagnostics := composeDocumentsWithPolicy(policy, parseCompositionDocument(t, "source-1", `
not-a-sequence: null
catalog:
@@ -454,7 +653,7 @@ catalog:
}
composed, diagnostics = composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}},
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "records: []\n"),
parseCompositionDocument(t, "source-2", "records: []\n"),
)
@@ -471,7 +670,7 @@ catalog:
func TestComposeDocumentsKeepsUnregisteredSequenceConflicts(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/registered": "id"}},
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("registered")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", "unregistered: [one]\n"),
parseCompositionDocument(t, "source-2", "unregistered: [two]\n"),
)
@@ -491,7 +690,7 @@ func TestComposeDocumentsKeepsNonSequenceRegisteredOverlapsConflicting(t *testin
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, diagnostics := composeDocumentsWithPolicy(
sequencePolicy{identityFields: map[string]string{"/records": "id"}},
mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}}),
parseCompositionDocument(t, "source-1", test.first),
parseCompositionDocument(t, "source-2", test.second),
)

View File

@@ -34,7 +34,7 @@ func TestDestinationPlacementCountsAgainstSharedNodeBudget(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
accounting := &importAccounting{nodes: limits.nodes - perInput + test.delta}
composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{
composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: empty, destination: destination,
}}}, 1, nil, accounting)
if test.wantError {
@@ -62,7 +62,7 @@ func TestMultipleMountedInputsSharePlacementCharges(t *testing.T) {
perInput := countInputNodes(&document.Node{Kind: document.Mapping}) + destinationWrapperNodeCount(Keys("collection", "one"))
charge := len(inputs) * perInput
accounting := &importAccounting{nodes: limits.nodes - charge}
composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting)
composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting)
if len(diagnostics) != 0 || composition == nil || accounting.nodes != limits.nodes || composition.inputNodes != charge {
t.Fatalf("shared placement accounting = %#v, %#v, accounting %#v; want charge %d", composition, diagnostics, accounting, charge)
}
@@ -107,7 +107,7 @@ id: parent
} {
t.Run(test.name, func(t *testing.T) {
accounting := &importAccounting{nodes: limits.nodes - wantNodes + test.delta}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{
composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: File(mainPath), imports: true, destination: destination, placementMode: placementModeSequenceItem,
}, {
source: Mapping("other", map[string]any{"id": "other"}), destination: otherDestination, placementMode: placementModeSequenceItem,
@@ -160,7 +160,7 @@ func TestImportedEmptyBodiesChargeEachDestinationWrapper(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
accounting := &importAccounting{nodes: limits.nodes - rawNodes - placementNodes + test.delta}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{
composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{
source: mainSource, imports: true, destination: destination,
}}}, 1, nil, accounting)
if test.wantError {
@@ -387,7 +387,7 @@ func TestNoSourceAndRootPlacementAddNoWrapperCharges(t *testing.T) {
t.Parallel()
accounting := &importAccounting{}
composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{
composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{
{source: NoSource(), destination: Keys("absent")},
{source: Mapping("root", map[string]any{})},
}}, 1, nil, accounting)
@@ -431,7 +431,13 @@ func TestSelectedOverlayAndCLIUsePlacementNodeCharges(t *testing.T) {
overrides: []Override{override},
}
accounting := &importAccounting{nodes: limits.nodes - baseNodes - overrideNodes + 1}
_, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting)
input, diagnostics := assembleLayeredInput(request, assembledInput{}, accounting)
if len(diagnostics) != 0 {
t.Fatalf("assembly diagnostics = %#v", diagnostics)
}
_, diagnostics = preparePostStructuralOverrides(overridePreparationInput{
root: input.document, inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}, input.sources, request.overrides, nil, limits)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeOverrideLimitExceeded {
t.Fatalf("override diagnostics = %#v", diagnostics)
}

View File

@@ -197,8 +197,8 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
t.Parallel()
definition := Definition[map[string]any]{
Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }},
sequenceRegistrations: []sequenceRegistration{{path: Keys("catalog", "entries"), identityField: "id"}},
Version: &VersionRule{Path: Keys("schema", "version"), Check: func(Version, *Validation) error { return nil }},
sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("catalog"), literalSequenceSelector("entries")}, identityFields: []string{"id"}}},
}
item := func(label string, value map[string]any) Input {
return Input{Source: Mapping(label, value), Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem}
@@ -209,7 +209,7 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
main := item("main", map[string]any{"id": "main", "schema": map[string]any{"version": "v1"}})
t.Run("same assigned main path is protected", func(t *testing.T) {
_, err := LoadLayered(main, testSecretBinding(), options(item("underlay", map[string]any{"id": "underlay", "schema": nil})))
_, err := LoadLayered(main, testSecretBinding(), options(item("underlay", map[string]any{"id": "main", "schema": nil})))
if got := groupedErrorCode(t, err); got != CodeUnderlayVersion {
t.Fatalf("protected item claim code = %s, want %s", got, CodeUnderlayVersion)
}
@@ -224,7 +224,7 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) {
t.Fatalf("unrelated appended peer rejected: %v", err)
}
entries := layeredMapValue(t, testLoaded(loaded).Config, "catalog", "entries").([]any)
if len(entries) != 1 || entries[0].(map[string]any)["id"] != "main" {
if len(entries) != 3 || entries[0].(map[string]any)["id"] != "first" || entries[1].(map[string]any)["id"] != "peer" || entries[2].(map[string]any)["id"] != "main" {
t.Fatalf("main item projection = %#v", entries)
}
})

View File

@@ -9,29 +9,104 @@ import (
const directiveKey = "configkit"
type rootDirectiveContext uint8
const (
rootDirectiveContextImportsOnly rootDirectiveContext = iota + 1
rootDirectiveContextSource
rootDirectiveContextOverlay
rootDirectiveContextLayered
)
type rootDirectiveExtraction struct {
body *document.Node
imports []importDeclarationSpec
sequences []sequenceDeclarationSpec
importsPresent bool
sequencesPresent bool
}
func splitImportDirectives(root *document.Node) (*document.Node, []importDeclarationSpec, []Diagnostic) {
extracted, diagnostics := extractRootDirectives(root, rootDirectiveContextImportsOnly)
if len(diagnostics) != 0 {
return nil, nil, diagnostics
}
return extracted.body, extracted.imports, nil
}
func extractRootDirectives(root *document.Node, context rootDirectiveContext) (rootDirectiveExtraction, []Diagnostic) {
if root == nil {
return rootDirectiveExtraction{}, nil
}
body := document.Clone(root)
var declarations []importDeclarationSpec
directiveIndex := -1
for index, entry := range body.Mapping {
if entry.Key != directiveKey {
continue
}
directiveIndex = index
parsed, diagnostics := parseImportSpecificationDirective(entry.Value, Keys(directiveKey))
extracted, diagnostics := parseRootDirectiveBlock(entry.Value, Keys(directiveKey), context)
if len(diagnostics) != 0 {
return nil, nil, diagnostics
return rootDirectiveExtraction{}, diagnostics
}
declarations = parsed
break
}
if directiveIndex >= 0 {
body.Mapping = append(body.Mapping[:directiveIndex], body.Mapping[directiveIndex+1:]...)
extracted.body = body
if directiveIndex >= 0 {
extracted.body.Mapping = append(extracted.body.Mapping[:directiveIndex], extracted.body.Mapping[directiveIndex+1:]...)
}
if diagnostics := nestedDirectiveDiagnostics(extracted.body, Path{}); len(diagnostics) != 0 {
return rootDirectiveExtraction{}, diagnostics
}
return extracted, nil
}
if diagnostics := nestedDirectiveDiagnostics(body, Path{}); len(diagnostics) != 0 {
return nil, nil, diagnostics
return rootDirectiveExtraction{}, diagnostics
}
return body, declarations, nil
return rootDirectiveExtraction{body: body}, nil
}
func parseRootDirectiveBlock(node *document.Node, directivePath Path, context rootDirectiveContext) (rootDirectiveExtraction, []Diagnostic) {
if node == nil || node.Kind != document.Mapping {
return rootDirectiveExtraction{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, directivePath, "configuration directive must be a mapping", node)}
}
var extracted rootDirectiveExtraction
var diagnostics []Diagnostic
for _, entry := range node.Mapping {
fieldPath := directivePath.Key(entry.Key)
switch entry.Key {
case "imports":
extracted.importsPresent = true
if context == rootDirectiveContextOverlay {
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeOverlayDirective, fieldPath, "overlays cannot contain imports", entry.Value))
continue
}
if context != rootDirectiveContextImportsOnly && context != rootDirectiveContextSource && context != rootDirectiveContextLayered {
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "directive extraction context is invalid", entry.Value))
continue
}
parsed, issues := parseImportDeclarations(entry.Value, fieldPath)
extracted.imports = append(extracted.imports, parsed...)
diagnostics = append(diagnostics, issues...)
case "sequences":
extracted.sequencesPresent = true
if context == rootDirectiveContextImportsOnly {
diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "sequence declarations are not enabled in this extraction context", entry.Value))
continue
}
if context != rootDirectiveContextSource && context != rootDirectiveContextOverlay && context != rootDirectiveContextLayered {
diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "directive extraction context is invalid", entry.Value))
continue
}
parsed, issues := parseSequenceDeclarationDirective(entry.Value, fieldPath)
extracted.sequences = append(extracted.sequences, parsed...)
diagnostics = append(diagnostics, issues...)
default:
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value))
}
}
if len(diagnostics) != 0 {
return rootDirectiveExtraction{}, canonicalDiagnostics(diagnostics)
}
return extracted, nil
}
func nestedDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic {
@@ -91,10 +166,11 @@ func fragmentVersionClaimWithIndex(root *document.Node, rule *VersionRule, sourc
return current
}
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found {
if index < assigned.start || index >= assigned.end {
projected, ok := assigned.inputForOutput(index)
if !ok {
return nil
}
index -= assigned.start
index = projected
}
if index < 0 || index >= len(current.Sequence) {
return nil
@@ -143,24 +219,28 @@ func nodeImportDiagnostic(code Code, path Path, message string, node *document.N
return Diagnostic{Code: code, Severity: SeverityError, Path: path, Message: message, Primary: &location}
}
func overlayDirectiveDiagnostics(node *document.Node, current Path) []Diagnostic {
func overlayDirectiveDiagnostics(node *document.Node, current Path, allowSequences ...bool) []Diagnostic {
sequencesAllowed := len(allowSequences) != 0 && allowSequences[0]
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
childPath := current.Key(entry.Key)
if entry.Key == directiveKey {
if sequencesAllowed && len(current.segments) == 0 && entry.Value != nil && entry.Value.Kind == document.Mapping && len(entry.Value.Mapping) == 1 && entry.Value.Mapping[0].Key == "sequences" {
continue
}
location := publicLocation(entry.KeyLocation)
return []Diagnostic{{
Code: CodeOverlayDirective, Severity: SeverityError, Path: childPath,
Message: "overlays cannot contain configuration directives", Primary: &location,
Message: "overlays may contain only root collection sequence declarations", Primary: &location,
}}
}
if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 {
if diagnostics := overlayDirectiveDiagnostics(entry.Value, childPath, sequencesAllowed); len(diagnostics) != 0 {
return diagnostics
}
}
} else if node.Kind == document.Sequence {
for index, child := range node.Sequence {
if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index)); len(diagnostics) != 0 {
if diagnostics := overlayDirectiveDiagnostics(child, current.Index(index), sequencesAllowed); len(diagnostics) != 0 {
return diagnostics
}
}
@@ -189,3 +269,72 @@ func disabledDirectiveDiagnostics(node *document.Node, path Path) []Diagnostic {
}
return diagnostics
}
func disabledRootImportDiagnostics(node *document.Node) []Diagnostic {
if node == nil || node.Kind != document.Mapping {
return nil
}
var rootDirective *document.Node
var rootLocation document.Location
var foundRootDirective bool
for index := range node.Mapping {
if node.Mapping[index].Key == directiveKey {
rootDirective = node.Mapping[index].Value
rootLocation = node.Mapping[index].KeyLocation
foundRootDirective = true
break
}
}
if foundRootDirective {
if rootDirective == nil || rootDirective.Kind != document.Mapping {
return disabledDirectiveDiagnostic(rootLocation)
}
for _, directive := range rootDirective.Mapping {
if directive.Key == "imports" {
return disabledDirectiveDiagnostic(rootLocation)
}
}
}
for _, entry := range node.Mapping {
if entry.Key == directiveKey {
continue
}
if diagnostic := disabledNestedDirectiveDiagnostics(entry.Value, Path{}.Key(entry.Key)); len(diagnostic) != 0 {
return diagnostic
}
}
return nil
}
func disabledNestedDirectiveDiagnostics(node *document.Node, path Path) []Diagnostic {
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
childPath := path.Key(entry.Key)
if entry.Key == directiveKey {
location := Location{SourceID: entry.KeyLocation.SourceID, Line: entry.KeyLocation.Line, Column: entry.KeyLocation.Column}
return []Diagnostic{{
Code: CodeImportDisabled, Severity: SeverityError, Path: childPath,
Message: "configuration directives require imports to be enabled", Primary: &location,
}}
}
if diagnostics := disabledNestedDirectiveDiagnostics(entry.Value, childPath); len(diagnostics) != 0 {
return diagnostics
}
}
} else if node.Kind == document.Sequence {
for index, child := range node.Sequence {
if diagnostics := disabledNestedDirectiveDiagnostics(child, path.Index(index)); len(diagnostics) != 0 {
return diagnostics
}
}
}
return nil
}
func disabledDirectiveDiagnostic(location document.Location) []Diagnostic {
public := Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column}
return []Diagnostic{{
Code: CodeImportDisabled, Severity: SeverityError, Path: Keys(directiveKey),
Message: "configuration directives require imports to be enabled", Primary: &public,
}}
}

View File

@@ -0,0 +1,540 @@
package engine
import (
"errors"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
)
// Search is an immutable ordered description of possible host-file inputs.
// It does not access the filesystem until a later loading integration consumes it.
type Search struct {
entries []searchEntry
issues []string
}
// FilenameGroup is an immutable ordered group used by a directory search.
type FilenameGroup struct {
kind filenameGroupKind
names []string
issues []string
}
// DirectorySearchOptions controls the filename groups used by SearchDirectory.
// A nil Filenames slice selects the default groups; a non-nil empty slice selects none.
type DirectorySearchOptions struct {
Filenames []FilenameGroup
}
// SearchOptions controls whether a later source consumption requires a match.
type SearchOptions struct {
Required bool
}
// SearchConfig creates a source that selects the first matching host file when
// consumed. Description and option errors are reported during loading.
func SearchConfig(application string, search Search, supplied ...SearchOptions) Source {
captured := Search{entries: cloneSearchEntries(search.entries), issues: append([]string(nil), search.issues...)}
source := Source{kind: sourceSearch, search: &captured, searchApp: application}
if len(supplied) > 1 {
source.searchIssues = []string{"SearchConfig accepts at most one options value"}
} else if len(supplied) == 1 {
source.searchRequired = supplied[0].Required
}
return source
}
type filenameGroupKind uint8
const (
filenameGroupEmpty filenameGroupKind = iota
filenameGroupNamed
filenameGroupGeneric
filenameGroupCustom
)
type searchEntryKind uint8
const (
searchEntryPath searchEntryKind = iota + 1
searchEntryDirectory
)
type directoryBaseKind uint8
const (
directoryBaseLiteral directoryBaseKind = iota
directoryBaseUnixLocal
directoryBaseUnixEtc
directoryBaseUserConfig
directoryBaseHome
directoryBaseWorking
)
type searchEntry struct {
kind searchEntryKind
path string
baseKind directoryBaseKind
directory string
appendSlug bool
slugPrefix string
filenames []FilenameGroup
}
// NamedFilenames selects <application>.yml followed by <application>.yaml.
func NamedFilenames() FilenameGroup {
return FilenameGroup{kind: filenameGroupNamed}
}
// GenericFilenames selects config.yml, config.yaml, main.yml, and main.yaml.
func GenericFilenames() FilenameGroup {
return FilenameGroup{kind: filenameGroupGeneric}
}
// CustomFilenames selects literal filename components in the supplied order.
func CustomFilenames(names ...string) FilenameGroup {
group := FilenameGroup{kind: filenameGroupCustom, names: append([]string(nil), names...)}
for _, name := range group.names {
if !validFilenameComponent(name) {
group.issues = append(group.issues, "custom filename must be a usable host filename component")
}
}
return group
}
// SearchDirectory searches the supplied directory using the selected filename groups.
// With omitted options or nil Filenames, it selects Named then Generic filenames.
func SearchDirectory(directory string, supplied ...DirectorySearchOptions) Search {
search := Search{}
if len(supplied) > 1 {
search.issues = append(search.issues, "directory search accepts at most one options value")
}
var groups []FilenameGroup
if len(supplied) == 0 || supplied[0].Filenames == nil {
groups = []FilenameGroup{NamedFilenames(), GenericFilenames()}
} else {
groups = cloneFilenameGroups(supplied[0].Filenames)
}
search.entries = append(search.entries, searchEntry{
kind: searchEntryDirectory, baseKind: directoryBaseLiteral,
directory: directory, filenames: groups,
})
return search
}
// Paths describes literal host-file candidates in the supplied order.
func Paths(paths ...string) Search {
search := Search{entries: make([]searchEntry, 0, len(paths))}
for _, path := range append([]string(nil), paths...) {
search.entries = append(search.entries, searchEntry{kind: searchEntryPath, path: path})
}
return search
}
// Searches concatenates descriptions in order and snapshots their entries.
func Searches(searches ...Search) Search {
result := Search{}
for _, search := range append([]Search(nil), searches...) {
result.entries = append(result.entries, cloneSearchEntries(search.entries)...)
result.issues = append(result.issues, search.issues...)
}
return result
}
// UnixSystemDirectories describes the conventional Unix system directories.
func UnixSystemDirectories() Search {
namedGeneric := []FilenameGroup{NamedFilenames(), GenericFilenames()}
return Search{entries: []searchEntry{
{kind: searchEntryDirectory, baseKind: directoryBaseUnixLocal, appendSlug: true, filenames: cloneFilenameGroups(namedGeneric)},
{kind: searchEntryDirectory, baseKind: directoryBaseUnixEtc, appendSlug: true, filenames: cloneFilenameGroups(namedGeneric)},
{kind: searchEntryDirectory, baseKind: directoryBaseUnixEtc, filenames: []FilenameGroup{NamedFilenames()}},
}}
}
// UserConfigDirectory describes the Go user-configuration directory searches.
func UserConfigDirectory() Search {
return Search{entries: []searchEntry{
{kind: searchEntryDirectory, baseKind: directoryBaseUserConfig, appendSlug: true, filenames: defaultFilenameGroups()},
{kind: searchEntryDirectory, baseKind: directoryBaseUserConfig, filenames: []FilenameGroup{NamedFilenames()}},
}}
}
// HomeDirectory describes the Go home-directory searches.
func HomeDirectory() Search {
return Search{entries: []searchEntry{
{kind: searchEntryDirectory, baseKind: directoryBaseHome, appendSlug: true, slugPrefix: ".", filenames: defaultFilenameGroups()},
{kind: searchEntryDirectory, baseKind: directoryBaseHome, filenames: []FilenameGroup{NamedFilenames()}},
}}
}
// WorkingDirectory describes application-named files in the current directory.
func WorkingDirectory() Search {
return Search{entries: []searchEntry{{
kind: searchEntryDirectory, baseKind: directoryBaseWorking,
filenames: []FilenameGroup{NamedFilenames()},
}}}
}
func defaultFilenameGroups() []FilenameGroup {
return []FilenameGroup{NamedFilenames(), GenericFilenames()}
}
func cloneFilenameGroups(groups []FilenameGroup) []FilenameGroup {
if groups == nil {
return nil
}
cloned := make([]FilenameGroup, len(groups))
for i, group := range groups {
cloned[i] = FilenameGroup{
kind: group.kind, names: append([]string(nil), group.names...),
issues: append([]string(nil), group.issues...),
}
}
return cloned
}
func cloneSearchEntries(entries []searchEntry) []searchEntry {
if entries == nil {
return nil
}
cloned := make([]searchEntry, len(entries))
for i, entry := range entries {
cloned[i] = entry
cloned[i].filenames = cloneFilenameGroups(entry.filenames)
}
return cloned
}
func validateSearch(search Search, application string) []Diagnostic {
var diagnostics []Diagnostic
for _, issue := range search.issues {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, issue))
}
slugNeeded := false
for _, entry := range search.entries {
switch entry.kind {
case searchEntryPath:
if entry.path == "" {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "search path must not be empty"))
}
case searchEntryDirectory:
if entry.baseKind == directoryBaseLiteral && entry.directory == "" {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "directory search path must not be empty"))
}
if entry.appendSlug {
slugNeeded = true
}
for _, group := range entry.filenames {
switch group.kind {
case filenameGroupEmpty, filenameGroupGeneric:
case filenameGroupNamed:
slugNeeded = true
case filenameGroupCustom:
default:
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "filename group is invalid"))
}
for _, issue := range group.issues {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, issue))
}
}
default:
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "search description is invalid"))
}
}
if slugNeeded && !validFilenameComponent(application) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "application name must be a usable host filename component"))
}
return canonicalDiagnostics(diagnostics)
}
func validFilenameComponent(value string) bool {
if value == "" || value == "." || value == ".." || strings.IndexByte(value, 0) >= 0 {
return false
}
if filepath.IsAbs(value) || filepath.VolumeName(value) != "" || filepath.Base(value) != value {
return false
}
if runtime.GOOS == "windows" {
if !filepath.IsLocal(value) || strings.ContainsAny(value, `<>:"|?*`) || strings.HasSuffix(value, ".") || strings.HasSuffix(value, " ") {
return false
}
for _, character := range value {
if character < 32 {
return false
}
}
}
return true
}
var errUnixSystemPresetUnavailable = errors.New("Unix system directories are unavailable on this platform")
type discoveryDirectoryHelpers struct {
userConfigDir func() (string, error)
userHomeDir func() (string, error)
workingDir func() (string, error)
unixSupported func() bool
}
type searchCandidateIterator struct {
search Search
application string
helpers discoveryDirectoryHelpers
entryIndex int
groupIndex int
nameIndex int
directory string
dirReady bool
}
func newSearchCandidateIterator(search Search, application string, helpers discoveryDirectoryHelpers) (*searchCandidateIterator, []Diagnostic) {
if diagnostics := validateSearch(search, application); len(diagnostics) != 0 {
return nil, diagnostics
}
defaults := defaultDiscoveryDirectoryHelpers()
if helpers.userConfigDir == nil {
helpers.userConfigDir = defaults.userConfigDir
}
if helpers.userHomeDir == nil {
helpers.userHomeDir = defaults.userHomeDir
}
if helpers.workingDir == nil {
helpers.workingDir = defaults.workingDir
}
if helpers.unixSupported == nil {
helpers.unixSupported = defaults.unixSupported
}
return &searchCandidateIterator{search: search, application: application, helpers: helpers}, nil
}
func defaultDiscoveryDirectoryHelpers() discoveryDirectoryHelpers {
return discoveryDirectoryHelpers{
userConfigDir: os.UserConfigDir,
userHomeDir: os.UserHomeDir,
workingDir: os.Getwd,
unixSupported: unixSystemPresetSupported,
}
}
// next expands one candidate at a time, resolving only the directory currently
// reached. Directory filenames are visited before moving to the next directory.
func (iterator *searchCandidateIterator) next() (string, bool, error) {
for {
if iterator.dirReady {
entry := iterator.search.entries[iterator.entryIndex]
for iterator.groupIndex < len(entry.filenames) {
group := entry.filenames[iterator.groupIndex]
count := filenameGroupLength(group)
if iterator.nameIndex < count {
name := filenameGroupAt(group, iterator.nameIndex, iterator.application)
iterator.nameIndex++
return filepath.Join(iterator.directory, name), true, nil
}
iterator.groupIndex++
iterator.nameIndex = 0
}
iterator.dirReady = false
iterator.entryIndex++
}
if iterator.entryIndex >= len(iterator.search.entries) {
return "", false, nil
}
entry := iterator.search.entries[iterator.entryIndex]
if entry.kind == searchEntryPath {
iterator.entryIndex++
path, err := iterator.absoluteCandidate(entry.path)
if err != nil {
return "", false, err
}
return path, true, nil
}
if !directoryHasCandidates(entry.filenames) {
iterator.entryIndex++
continue
}
directory, err := iterator.resolveDirectory(entry)
if err != nil {
return "", false, err
}
iterator.directory = directory
iterator.groupIndex = 0
iterator.nameIndex = 0
iterator.dirReady = true
}
}
func (iterator *searchCandidateIterator) absoluteCandidate(path string) (string, error) {
if filepath.IsAbs(path) {
return filepath.Clean(path), nil
}
// Drive-relative and drive-root-relative Windows paths need native
// resolution; joining a rooted path to the whole working directory changes it.
if filepath.VolumeName(path) != "" || runtime.GOOS == "windows" && len(path) > 0 && os.IsPathSeparator(path[0]) {
return filepath.Abs(path)
}
working, err := iterator.helpers.workingDir()
if err != nil {
return "", err
}
return filepath.Clean(filepath.Join(working, path)), nil
}
func (iterator *searchCandidateIterator) resolveDirectory(entry searchEntry) (string, error) {
var base string
switch entry.baseKind {
case directoryBaseLiteral:
base = entry.directory
case directoryBaseUnixLocal:
if !iterator.helpers.unixSupported() {
return "", errUnixSystemPresetUnavailable
}
base = "/usr/local/etc"
case directoryBaseUnixEtc:
if !iterator.helpers.unixSupported() {
return "", errUnixSystemPresetUnavailable
}
base = "/etc"
case directoryBaseUserConfig:
var err error
base, err = iterator.helpers.userConfigDir()
if err != nil {
return "", err
}
case directoryBaseHome:
var err error
base, err = iterator.helpers.userHomeDir()
if err != nil {
return "", err
}
case directoryBaseWorking:
var err error
base, err = iterator.helpers.workingDir()
if err != nil {
return "", err
}
default:
return "", errors.New("directory search base is invalid")
}
if entry.appendSlug {
base = filepath.Join(base, entry.slugPrefix+iterator.application)
}
if entry.baseKind == directoryBaseLiteral && !filepath.IsAbs(base) {
return iterator.absoluteCandidate(base)
}
return filepath.Clean(base), nil
}
func directoryHasCandidates(groups []FilenameGroup) bool {
for _, group := range groups {
if filenameGroupLength(group) != 0 {
return true
}
}
return false
}
func filenameGroupLength(group FilenameGroup) int {
switch group.kind {
case filenameGroupNamed:
return 2
case filenameGroupGeneric:
return 4
case filenameGroupCustom:
return len(group.names)
default:
return 0
}
}
func filenameGroupAt(group FilenameGroup, index int, application string) string {
switch group.kind {
case filenameGroupNamed:
if index == 0 {
return application + ".yml"
}
return application + ".yaml"
case filenameGroupGeneric:
return [...]string{"config.yml", "config.yaml", "main.yml", "main.yaml"}[index]
case filenameGroupCustom:
return group.names[index]
default:
return ""
}
}
type discoveryHostProbe struct {
evalSymlinks func(string) (string, error)
stat func(string) (fs.FileInfo, error)
}
func defaultDiscoveryHostProbe() discoveryHostProbe {
return discoveryHostProbe{evalSymlinks: filepath.EvalSymlinks, stat: os.Stat}
}
// resolveSearchInput associates a terminal candidate failure with the same file
// evidence as ordinary acquisition. Exhaustion and base failures have no file.
func resolveSearchInput(source Source, sourceID string, helpers discoveryDirectoryHelpers, probe discoveryHostProbe) (Source, *SourceInfo, []Diagnostic) {
selected, diagnostics := resolveSearchSource(source, source.search, source.searchApp, source.searchRequired, helpers, probe)
if len(diagnostics) == 0 || selected.kind != sourceFile {
return selected, nil, diagnostics
}
info := &SourceInfo{ID: sourceID, Label: selected.label, Path: selected.path, Layer: LayerBase, Kind: SourceKindFile}
for index := range diagnostics {
diagnostics[index].Primary = &Location{SourceID: sourceID}
}
return selected, info, diagnostics
}
// resolveSearchSource selects one ordinary file source. A nil description leaves
// an existing source untouched; the selected file is read later by readSource.
// A terminal candidate failure returns its file descriptor for error evidence.
func resolveSearchSource(source Source, description *Search, application string, required bool, helpers discoveryDirectoryHelpers, probe discoveryHostProbe) (Source, []Diagnostic) {
if description == nil {
return source, nil
}
iterator, diagnostics := newSearchCandidateIterator(*description, application, helpers)
if len(diagnostics) != 0 {
return Source{}, diagnostics
}
defaults := defaultDiscoveryHostProbe()
if probe.evalSymlinks == nil {
probe.evalSymlinks = defaults.evalSymlinks
}
if probe.stat == nil {
probe.stat = defaults.stat
}
for {
candidate, ok, err := iterator.next()
if err != nil {
if errors.Is(err, errUnixSystemPresetUnavailable) {
return Source{}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "Unix system directory search is unavailable on this platform")}
}
return Source{}, []Diagnostic{classifySourceError(err, "")}
}
if !ok {
if required {
return Source{}, []Diagnostic{sourceDiagnostic(CodeSourceMissing, "", "required configuration search found no file")}
}
return NoSource(), nil
}
canonical, err := probe.evalSymlinks(candidate)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
return File(candidate), []Diagnostic{classifySourceError(err, "")}
}
if _, err := probe.stat(canonical); err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
failed := File(candidate)
failed.path = canonical
return failed, []Diagnostic{classifySourceError(err, "")}
}
return File(candidate), nil
}
}

View File

@@ -0,0 +1,430 @@
package engine
import (
"errors"
"os"
"path/filepath"
"reflect"
"runtime"
"testing"
)
type discoveryLoadConfig struct {
Service struct {
Settings map[string]string `yaml:"settings"`
} `yaml:"service"`
Settings map[string]string `yaml:"settings"`
Value string `yaml:"value"`
}
func TestSearchLoadMatchesExplicitFileThroughImportsAndContext(t *testing.T) {
t.Parallel()
root := t.TempDir()
mainPath := writeImportFile(t, root, "app.yml", `configkit:
imports: [{path: fragments/extra.yml}]
settings: {main: selected}
`)
importPath := writeImportFile(t, root, "fragments/extra.yml", "settings: {imported: value}\n")
search := SearchConfig("app", SearchDirectory(root))
options := Options[discoveryLoadConfig]{
Imports: true, Filesystem: &FilesystemContext{Root: root}, destination: Keys("service"),
}
searched, err := testLoad(search, options)
if err != nil {
t.Fatal(err)
}
explicit, err := testLoad(File(mainPath), options)
if err != nil {
t.Fatal(err)
}
assertEquivalentDiscoveryLoads(t, searched.Config, searched.Inspection, explicit.Config, explicit.Inspection,
Keys("service", "settings", "main"), Keys("service", "settings", "imported"))
sources := searched.Sources()
if len(sources) != 2 || sources[0].Kind != SourceKindFile || sources[0].Path != canonicalTestPath(t, mainPath) || sources[1].Path != canonicalTestPath(t, importPath) {
t.Fatalf("searched source evidence = %#v", sources)
}
}
func TestSearchLayeredMainAndUnderlayMatchExplicitFiles(t *testing.T) {
t.Parallel()
root := t.TempDir()
mainPath := writeImportFile(t, root, "main.yml", "settings: {main: value}\n")
underlayPath := writeImportFile(t, root, "underlay/base.yml", `configkit:
imports: [{path: extra.yml}]
settings: {underlay: value}
`)
importPath := writeImportFile(t, root, "underlay/extra.yml", "settings: {imported: value}\n")
mainSearch := SearchConfig("", SearchDirectory(root, DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("main.yml")}}))
underlaySearch := SearchConfig("", SearchDirectory(filepath.Join(root, "underlay"), DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("base.yml")}}))
load := func(main, underlay Source) (*Loaded[discoveryLoadConfig], error) {
return LoadLayered(
Input{Source: main, Imports: true}, testSecretBinding(),
LayeredOptions[discoveryLoadConfig]{Underlays: []Underlay{{Inputs: []Input{{Source: underlay, Imports: true}}}}},
)
}
searched, err := load(mainSearch, underlaySearch)
if err != nil {
t.Fatal(err)
}
explicit, err := load(File(mainPath), File(underlayPath))
if err != nil {
t.Fatal(err)
}
assertEquivalentDiscoveryLoads(t, searched.Config, searched.Inspection, explicit.Config, explicit.Inspection,
Keys("settings", "main"), Keys("settings", "underlay"), Keys("settings", "imported"))
sources := searched.Inspection.Sources()
if len(sources) != 3 || sources[0].ID != "source-1" || sources[0].Layer != LayerBase || sources[1].ID != "source-2" || sources[1].Layer != LayerUnderlay || sources[2].Path != canonicalTestPath(t, importPath) || sources[2].UnderlayIndex != 1 {
t.Fatalf("layered search source evidence = %#v", sources)
}
}
func TestOptionalSearchExhaustionMatchesNoSource(t *testing.T) {
t.Parallel()
root := t.TempDir()
missing := filepath.Join(root, "missing.yml")
defaults := func() discoveryLoadConfig {
return discoveryLoadConfig{Value: "default"}
}
options := Options[discoveryLoadConfig]{
Definition: Definition[discoveryLoadConfig]{Defaults: defaults},
Imports: true, Filesystem: &FilesystemContext{Root: root},
}
searched, err := testLoad(SearchConfig("", Paths(missing)), options)
if err != nil {
t.Fatal(err)
}
explicit, err := testLoad(NoSource(), options)
if err != nil {
t.Fatal(err)
}
assertEquivalentDiscoveryLoads(t, searched.Config, searched.Inspection, explicit.Config, explicit.Inspection, Keys("value"))
if len(searched.Sources()) != 0 {
t.Fatalf("optional exhausted search created sources: %#v", searched.Sources())
}
}
func TestSearchSelectedFileFailuresDoNotFallBack(t *testing.T) {
t.Parallel()
t.Run("malformed YAML", func(t *testing.T) {
first := writeImportFile(t, t.TempDir(), "first.yml", ": malformed\n")
writeImportFile(t, filepath.Dir(first), "later.yml", "value: valid\n")
loaded, err := testLoad[discoveryLoadConfig](SearchConfig("", Paths(first, filepath.Join(filepath.Dir(first), "later.yml"))), Options[discoveryLoadConfig]{})
if loaded != nil || err == nil {
t.Fatalf("malformed selected source = %#v, %v", loaded, err)
}
})
t.Run("version check", func(t *testing.T) {
root := t.TempDir()
first := writeImportFile(t, root, "first.yml", "version: rejected\n")
writeImportFile(t, root, "later.yml", "version: accepted\n")
var checked []string
loaded, err := testLoad[discoveryLoadConfig](SearchConfig("", Paths(first, filepath.Join(root, "later.yml"))), Options[discoveryLoadConfig]{
SourceVersionMode: sourceVersionModeIfPresent,
Definition: Definition[discoveryLoadConfig]{Version: &VersionRule{
Path: Keys("version"),
Check: func(version Version, _ *Validation) error {
checked = append(checked, version.Text)
if version.Text == "rejected" {
return errors.New("version rejected")
}
return nil
},
}},
})
if loaded != nil || err == nil || !reflect.DeepEqual(checked, []string{"rejected"}) {
t.Fatalf("version failure search = %#v, %v; checked %#v", loaded, err, checked)
}
})
t.Run("decode", func(t *testing.T) {
root := t.TempDir()
first := writeImportFile(t, root, "first.yml", "value: not-an-integer\n")
writeImportFile(t, root, "later.yml", "value: 3\n")
type config struct {
Value int `yaml:"value"`
}
loaded, err := testLoad[config](SearchConfig("", Paths(first, filepath.Join(root, "later.yml"))), Options[config]{})
if loaded != nil || err == nil {
t.Fatalf("decode failure search = %#v, %v", loaded, err)
}
})
}
func TestSearchProbeFailureRetainsFileEvidence(t *testing.T) {
t.Parallel()
root := t.TempDir()
main := writeImportFile(t, root, "main.yml", "value: main\n")
underlay := writeImportFile(t, root, "underlay.yml", "settings: {first: underlay}\n")
bad := filepath.Join(main, "child.yml")
later := writeImportFile(t, root, "later.yml", "value: later\n")
missing := filepath.Join(root, "missing.yml")
for _, entry := range []struct {
name string
id string
load func(Source) (*Loaded[discoveryLoadConfig], error)
}{
{"ordinary", "source-1", func(source Source) (*Loaded[discoveryLoadConfig], error) {
return Load(source, testSecretBinding(), Options[discoveryLoadConfig]{destination: Keys("settings")})
}},
{"underlay after acquired sources", "source-3", func(source Source) (*Loaded[discoveryLoadConfig], error) {
return LoadLayered(Input{Source: File(main)}, testSecretBinding(), LayeredOptions[discoveryLoadConfig]{
Underlays: []Underlay{{Inputs: []Input{
{Source: File(underlay)}, {Source: source, Destination: Keys("service", "settings")},
}}},
})
}},
} {
t.Run(entry.name, func(t *testing.T) {
_, err := entry.load(File(bad))
var explicit *LoadError
if !errors.As(err, &explicit) || len(explicit.Diagnostics()) != 1 {
t.Fatalf("explicit failed file = %v", err)
}
if explicit.Diagnostics()[0].Code == CodeSourceMissing {
t.Skip("host classifies a non-directory ancestor as absence; controlled probe tests cover access failures")
}
result, err := entry.load(SearchConfig("", Paths(missing, bad, later)))
var searched *LoadError
if result != nil || !errors.As(err, &searched) {
t.Fatalf("failed search = %#v, %v", result, err)
}
if !reflect.DeepEqual(searched.Sources(), explicit.Sources()) || !reflect.DeepEqual(searched.Diagnostics(), explicit.Diagnostics()) {
t.Fatalf("search failure differs from explicit file:\nsources %#v / %#v\ndiagnostics %#v / %#v", searched.Sources(), explicit.Sources(), searched.Diagnostics(), explicit.Diagnostics())
}
primary := searched.Diagnostics()[0].Primary
if primary == nil || primary.SourceID != entry.id {
t.Fatalf("failed candidate location = %#v, want %s", primary, entry.id)
}
})
}
}
func TestSearchValidationPrecedesCandidateProbing(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := writeImportFile(t, root, "first.yml", "value: selected\n")
_, err := testLoad[discoveryLoadConfig](SearchConfig("", Searches(Paths(first), Paths(""))), Options[discoveryLoadConfig]{})
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("invalid later search error = %v, want *LoadError", err)
}
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != CodeOptionsInvalid {
t.Fatalf("invalid later search component diagnostics = %#v", loadErr.Diagnostics())
}
malformed := writeImportFile(t, t.TempDir(), "main.yml", ": malformed\n")
loaded, err := LoadLayered(
Input{Source: SearchConfig("", Paths(malformed))}, testSecretBinding(),
LayeredOptions[discoveryLoadConfig]{Underlays: []Underlay{{Inputs: []Input{{Source: SearchConfig("", Paths(""))}}}}},
)
if loaded != nil || !errors.As(err, &loadErr) {
t.Fatalf("invalid later underlay search = %#v, %v", loaded, err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code != CodeOptionsInvalid {
t.Fatalf("layered validation acquired main before rejecting underlay: %#v", loadErr.Diagnostics())
}
}
}
func TestSearchContextCompatibilityUsesSelectedOutcome(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := writeImportFile(t, root, "first.yml", "value: first\n")
writeImportFile(t, root, "later.yml", "value: later\n")
search := SearchConfig("", Paths(first, filepath.Join(root, "later.yml")))
options := Options[discoveryLoadConfig]{Filesystem: &FilesystemContext{BaseDir: root, Root: root}}
_, err := testLoad[discoveryLoadConfig](search, options)
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("selected file with explicit base error = %v, want *LoadError", err)
}
if len(loadErr.Diagnostics()) == 0 || loadErr.Diagnostics()[0].Code != CodeOptionsInvalid {
t.Fatalf("selected file context diagnostics = %#v", loadErr.Diagnostics())
}
}
func TestSearchEmptyContextUsesSelectedOutcome(t *testing.T) {
t.Parallel()
root := t.TempDir()
selected := writeImportFile(t, root, "selected.yml", "value: selected\n")
missing := filepath.Join(root, "missing.yml")
context := &FilesystemContext{}
for _, entry := range []struct {
name string
load func(Source) (*Loaded[discoveryLoadConfig], error)
}{
{"ordinary", func(source Source) (*Loaded[discoveryLoadConfig], error) {
return Load(source, testSecretBinding(), Options[discoveryLoadConfig]{Imports: true, Filesystem: context})
}},
{"layered main", func(source Source) (*Loaded[discoveryLoadConfig], error) {
return LoadLayered(Input{Source: source, Imports: true, Filesystem: context}, testSecretBinding(), LayeredOptions[discoveryLoadConfig]{})
}},
{"underlay", func(source Source) (*Loaded[discoveryLoadConfig], error) {
return LoadLayered(Input{Source: NoSource()}, testSecretBinding(), LayeredOptions[discoveryLoadConfig]{
Underlays: []Underlay{{Inputs: []Input{{Source: source, Imports: true, Filesystem: context}}}},
})
}},
} {
t.Run(entry.name, func(t *testing.T) {
explicit, err := entry.load(File(selected))
if err != nil {
t.Fatal(err)
}
searched, err := entry.load(SearchConfig("", Paths(missing, selected)))
if err != nil {
t.Fatal(err)
}
assertEquivalentDiscoveryLoads(t, searched.Config, searched.Inspection, explicit.Config, explicit.Inspection, Keys("value"))
if searched.Config.Value != "selected" {
t.Fatalf("selected value = %q", searched.Config.Value)
}
for _, source := range []Source{NoSource(), SearchConfig("", Paths(missing))} {
result, err := entry.load(source)
var failure *LoadError
if result != nil || !errors.As(err, &failure) || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != CodeOptionsInvalid {
t.Fatalf("absent input with empty context = %#v, %v", result, err)
}
}
})
}
}
func TestSearchSourceObservesFileChangesBetweenLoads(t *testing.T) {
t.Parallel()
root := t.TempDir()
file := writeImportFile(t, root, "config.yml", "value: first\n")
source := SearchConfig("", SearchDirectory(root, DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("config.yml")}}))
load := func() string {
loaded, err := testLoad[discoveryLoadConfig](source, Options[discoveryLoadConfig]{})
if err != nil {
t.Fatal(err)
}
return loaded.Config.Value
}
if value := load(); value != "first" {
t.Fatalf("first search value = %q", value)
}
if err := os.WriteFile(file, []byte("value: second\n"), 0o600); err != nil {
t.Fatal(err)
}
if value := load(); value != "second" {
t.Fatalf("updated search value = %q", value)
}
}
func TestSearchSourceObservesUserConfigEnvironmentChanges(t *testing.T) {
setBase := func(directory string) string {
t.Helper()
switch runtime.GOOS {
case "windows":
t.Setenv("AppData", directory)
case "darwin", "ios":
t.Setenv("HOME", directory)
case "plan9":
t.Setenv("home", directory)
default:
t.Setenv("XDG_CONFIG_HOME", directory)
}
base, err := os.UserConfigDir()
if err != nil {
t.Fatal(err)
}
return base
}
type config struct {
Value string `yaml:"value"`
}
firstBase := setBase(filepath.Join(t.TempDir(), "first"))
first := writeImportFile(t, firstBase, "app/app.yml", "value: first\n")
source := SearchConfig("app", UserConfigDirectory())
load := func() string {
loaded, err := testLoad[config](source, Options[config]{})
if err != nil {
t.Fatal(err)
}
return loaded.Config.Value
}
if value := load(); value != "first" {
t.Fatalf("first user-config search value = %q (%s)", value, first)
}
secondBase := setBase(filepath.Join(t.TempDir(), "second"))
second := writeImportFile(t, secondBase, "app/app.yml", "value: second\n")
if value := load(); value != "second" {
t.Fatalf("updated user-config search value = %q (%s)", value, second)
}
}
func TestSearchSourceObservesWorkingDirectoryChanges(t *testing.T) {
original, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := os.Chdir(original); err != nil {
t.Errorf("restore working directory: %v", err)
}
})
firstDir := t.TempDir()
secondDir := t.TempDir()
writeImportFile(t, firstDir, "app.yml", "value: first\n")
writeImportFile(t, secondDir, "app.yml", "value: second\n")
source := SearchConfig("app", WorkingDirectory())
load := func() string {
loaded, err := testLoad[discoveryLoadConfig](source, Options[discoveryLoadConfig]{})
if err != nil {
t.Fatal(err)
}
return loaded.Config.Value
}
if err := os.Chdir(firstDir); err != nil {
t.Fatal(err)
}
if value := load(); value != "first" {
t.Fatalf("first working-directory search value = %q", value)
}
if err := os.Chdir(secondDir); err != nil {
t.Fatal(err)
}
if value := load(); value != "second" {
t.Fatalf("updated working-directory search value = %q", value)
}
}
func assertEquivalentDiscoveryLoads[T any](t *testing.T, leftConfig T, left *Inspection, rightConfig T, right *Inspection, presencePaths ...Path) {
t.Helper()
if !reflect.DeepEqual(leftConfig, rightConfig) {
t.Fatalf("search config %#v differs from explicit file config %#v", leftConfig, rightConfig)
}
leftSources, rightSources := left.Sources(), right.Sources()
if !reflect.DeepEqual(leftSources, rightSources) {
t.Fatalf("search sources %#v differ from explicit file sources %#v", leftSources, rightSources)
}
leftMetadata, rightMetadata := left.Metadata(), right.Metadata()
for _, path := range presencePaths {
leftInfo, leftPresent := leftMetadata.Lookup(path)
rightInfo, rightPresent := rightMetadata.Lookup(path)
if leftPresent != rightPresent || !reflect.DeepEqual(leftInfo, rightInfo) {
t.Fatalf("search metadata at %s = %#v, %v; explicit metadata = %#v, %v", path.String(), leftInfo, leftPresent, rightInfo, rightPresent)
}
}
for _, source := range leftSources {
leftContext, leftOK := left.SourceContext(source.ID)
rightContext, rightOK := right.SourceContext(source.ID)
if leftOK != rightOK || leftContext.BaseDir != rightContext.BaseDir || (leftContext.FS != nil) != (rightContext.FS != nil) {
t.Fatalf("search context for %s = %#v, %v; explicit context = %#v, %v", source.ID, leftContext, leftOK, rightContext, rightOK)
}
}
}

View File

@@ -0,0 +1,5 @@
//go:build !aix && !android && !darwin && !dragonfly && !freebsd && !hurd && !illumos && !ios && !linux && !netbsd && !openbsd && !solaris
package engine
func unixSystemPresetSupported() bool { return false }

View File

@@ -0,0 +1,5 @@
//go:build aix || android || darwin || dragonfly || freebsd || hurd || illumos || ios || linux || netbsd || openbsd || solaris
package engine
func unixSystemPresetSupported() bool { return true }

View File

@@ -0,0 +1,519 @@
package engine
import (
"errors"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
func TestSearchDescriptionEmptyAndSlugRequirements(t *testing.T) {
t.Parallel()
if diagnostics := validateSearch(Search{}, ""); len(diagnostics) != 0 {
t.Fatalf("zero search diagnostics = %#v", diagnostics)
}
if diagnostics := validateSearch(Searches(), ""); len(diagnostics) != 0 {
t.Fatalf("empty concatenation diagnostics = %#v", diagnostics)
}
if diagnostics := validateSearch(Paths(), ""); len(diagnostics) != 0 {
t.Fatalf("empty path list diagnostics = %#v", diagnostics)
}
if diagnostics := validateSearch(Paths("relative/config.yml"), ""); len(diagnostics) != 0 {
t.Fatalf("literal path unexpectedly requires a slug: %#v", diagnostics)
}
generic := SearchDirectory("configs", DirectorySearchOptions{Filenames: []FilenameGroup{GenericFilenames()}})
if diagnostics := validateSearch(generic, ""); len(diagnostics) != 0 {
t.Fatalf("generic-only directory unexpectedly requires a slug: %#v", diagnostics)
}
if diagnostics := validateSearch(SearchDirectory("configs"), ""); len(diagnostics) == 0 || diagnostics[0].Code != CodeOptionsInvalid {
t.Fatalf("default named directory accepted an empty slug: %#v", diagnostics)
}
if diagnostics := validateSearch(UnixSystemDirectories(), "myapp"); len(diagnostics) != 0 {
t.Fatalf("valid preset slug diagnostics = %#v", diagnostics)
}
}
func TestSearchDescriptionSnapshotsInputs(t *testing.T) {
t.Parallel()
paths := []string{"first.yml", "second.yml"}
pathSearch := Paths(paths...)
paths[0] = "changed.yml"
customNames := []string{"one.yml", "two.yml"}
group := CustomFilenames(customNames...)
customNames[0] = "changed.yml"
groups := []FilenameGroup{group}
options := DirectorySearchOptions{Filenames: groups}
directorySearch := SearchDirectory("configs", options)
groups[0] = GenericFilenames()
options.Filenames[0] = NamedFilenames()
searches := []Search{pathSearch, directorySearch}
combined := Searches(searches...)
searches[0] = Search{}
if got := combined.entries[0].path; got != "first.yml" {
t.Fatalf("snapshotted path = %q", got)
}
gotNames := combined.entries[2].filenames[0].names
if len(gotNames) != 2 || gotNames[0] != "one.yml" || gotNames[1] != "two.yml" {
t.Fatalf("snapshotted custom names = %#v", gotNames)
}
if diagnostics := validateSearch(combined, ""); len(diagnostics) != 0 {
t.Fatalf("snapshot description diagnostics = %#v", diagnostics)
}
}
func TestDirectoryFilenameDefaultsAndExplicitEmpty(t *testing.T) {
t.Parallel()
omitted := SearchDirectory("configs")
if len(omitted.entries) != 1 || len(omitted.entries[0].filenames) != 2 || omitted.entries[0].filenames[0].kind != filenameGroupNamed || omitted.entries[0].filenames[1].kind != filenameGroupGeneric {
t.Fatalf("omitted groups = %#v", omitted.entries)
}
explicitNil := SearchDirectory("configs", DirectorySearchOptions{})
if len(explicitNil.entries[0].filenames) != 2 {
t.Fatalf("nil filename selection did not use defaults: %#v", explicitNil.entries[0].filenames)
}
explicitEmpty := SearchDirectory("configs", DirectorySearchOptions{Filenames: []FilenameGroup{}})
if explicitEmpty.entries[0].filenames == nil || len(explicitEmpty.entries[0].filenames) != 0 {
t.Fatalf("explicit empty selection lost its distinction: %#v", explicitEmpty.entries[0].filenames)
}
if len(validateSearch(explicitEmpty, "")) != 0 {
t.Fatal("explicit empty filename selection should need no slug")
}
}
func TestSearchDescriptionArityAndErrorConcatenation(t *testing.T) {
t.Parallel()
tooMany := SearchDirectory("configs", DirectorySearchOptions{}, DirectorySearchOptions{})
if diagnostics := validateSearch(tooMany, "app"); len(diagnostics) == 0 {
t.Fatal("multiple option values should be invalid")
}
invalid := Paths("")
combined := Searches(Paths("good.yml"), invalid)
if diagnostics := validateSearch(combined, ""); len(diagnostics) == 0 {
t.Fatal("concatenation lost a constructor error")
}
if diagnostics := validateSearch(Paths("good.yml", ""), ""); len(diagnostics) == 0 {
t.Fatal("later invalid path was hidden by an earlier component")
}
}
func TestSearchDescriptionPreservesFilenameGroupOrder(t *testing.T) {
t.Parallel()
groups := []FilenameGroup{GenericFilenames(), NamedFilenames(), CustomFilenames("override.yml")}
search := SearchDirectory("configs", DirectorySearchOptions{Filenames: groups})
got := search.entries[0].filenames
if len(got) != 3 || got[0].kind != filenameGroupGeneric || got[1].kind != filenameGroupNamed || got[2].names[0] != "override.yml" {
t.Fatalf("filename group order = %#v", got)
}
if got := SearchDirectory("configs", DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames()}}).entries[0].filenames[0].names; len(got) != 0 {
t.Fatalf("empty custom group = %#v", got)
}
}
func TestFilenameComponentValidation(t *testing.T) {
t.Parallel()
for _, value := range []string{"", ".", "..", "nested/name", "bad\x00name"} {
if validFilenameComponent(value) {
t.Errorf("validFilenameComponent(%q) = true", value)
}
}
if runtime.GOOS == "windows" {
for _, value := range []string{"CON", "CONIN$", "CONOUT$", "name.", "bad:name", "bad\x01name"} {
if validFilenameComponent(value) {
t.Errorf("Windows component %q unexpectedly accepted", value)
}
}
// Device names with extensions depend on the Windows version. Match
// Go's platform-aware decision instead of imposing a blanket ban.
for _, value := range []string{"nul.txt", "COM¹.log", "LPT³.txt", "CON.extra.yml"} {
if got, want := validFilenameComponent(value), filepath.IsLocal(value); got != want {
t.Errorf("Windows component %q validity = %v, want Go's %v", value, got, want)
}
}
} else {
for _, value := range []string{"CONIN$", "CONOUT$", "nul.txt"} {
if !validFilenameComponent(value) {
t.Errorf("Windows component restriction applied to this host: %q", value)
}
}
}
for _, value := range []string{"name with spaces", "naïve"} {
if !validFilenameComponent(value) {
t.Errorf("host-valid component %q rejected", value)
}
}
if diagnostics := validateSearch(SearchDirectory("configs", DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("bad/name")}}), ""); len(diagnostics) == 0 {
t.Fatal("invalid custom filename was not retained for validation")
}
}
func TestSearchCandidatesPreserveDirectoryAndGroupOrder(t *testing.T) {
t.Parallel()
root := t.TempDir()
helpers := discoveryDirectoryHelpers{workingDir: func() (string, error) { return root, nil }}
search := Searches(
Paths("first.yml"),
SearchDirectory("settings", DirectorySearchOptions{Filenames: []FilenameGroup{
GenericFilenames(), CustomFilenames("special.yml"),
}}),
Paths("last.yml"),
)
candidates := collectSearchCandidates(t, search, "app", helpers)
want := []string{
filepath.Join(root, "first.yml"),
filepath.Join(root, "settings", "config.yml"),
filepath.Join(root, "settings", "config.yaml"),
filepath.Join(root, "settings", "main.yml"),
filepath.Join(root, "settings", "main.yaml"),
filepath.Join(root, "settings", "special.yml"),
filepath.Join(root, "last.yml"),
}
assertCandidatePaths(t, candidates, want)
reversed := SearchDirectory("settings", DirectorySearchOptions{Filenames: []FilenameGroup{NamedFilenames(), GenericFilenames()}})
reversedCandidates := collectSearchCandidates(t, reversed, "app", helpers)
assertCandidatePaths(t, reversedCandidates[:2], []string{
filepath.Join(root, "settings", "app.yml"),
filepath.Join(root, "settings", "app.yaml"),
})
}
func TestSearchPresetCandidateLists(t *testing.T) {
t.Parallel()
helpers := discoveryDirectoryHelpers{
userConfigDir: func() (string, error) { return "/controlled/config", nil },
userHomeDir: func() (string, error) { return "/controlled/home", nil },
workingDir: func() (string, error) { return "/controlled/work", nil },
unixSupported: func() bool { return true },
}
unix := collectSearchCandidates(t, UnixSystemDirectories(), "app", helpers)
if len(unix) != 14 {
t.Fatalf("Unix preset candidate count = %d: %#v", len(unix), unix)
}
assertCandidatePaths(t, unix, []string{
"/usr/local/etc/app/app.yml", "/usr/local/etc/app/app.yaml", "/usr/local/etc/app/config.yml", "/usr/local/etc/app/config.yaml", "/usr/local/etc/app/main.yml", "/usr/local/etc/app/main.yaml",
"/etc/app/app.yml", "/etc/app/app.yaml", "/etc/app/config.yml", "/etc/app/config.yaml", "/etc/app/main.yml", "/etc/app/main.yaml",
"/etc/app.yml", "/etc/app.yaml",
})
user := collectSearchCandidates(t, UserConfigDirectory(), "app", helpers)
assertCandidatePaths(t, user, []string{
"/controlled/config/app/app.yml", "/controlled/config/app/app.yaml", "/controlled/config/app/config.yml", "/controlled/config/app/config.yaml", "/controlled/config/app/main.yml", "/controlled/config/app/main.yaml",
"/controlled/config/app.yml", "/controlled/config/app.yaml",
})
home := collectSearchCandidates(t, HomeDirectory(), "app", helpers)
assertCandidatePaths(t, home, []string{
"/controlled/home/.app/app.yml", "/controlled/home/.app/app.yaml", "/controlled/home/.app/config.yml", "/controlled/home/.app/config.yaml", "/controlled/home/.app/main.yml", "/controlled/home/.app/main.yaml",
"/controlled/home/app.yml", "/controlled/home/app.yaml",
})
working := collectSearchCandidates(t, WorkingDirectory(), "app", helpers)
assertCandidatePaths(t, working, []string{"/controlled/work/app.yml", "/controlled/work/app.yaml"})
}
func TestSearchCandidateExpansionIsLazyAndSkipsEmptyDirectories(t *testing.T) {
t.Parallel()
configCalls := 0
helpers := discoveryDirectoryHelpers{
userConfigDir: func() (string, error) {
configCalls++
return "/controlled/config", nil
},
workingDir: func() (string, error) { return "/controlled/work", nil },
unixSupported: func() bool { return true },
}
search := Searches(Paths("/selected.yml"), UserConfigDirectory())
iterator, diagnostics := newSearchCandidateIterator(search, "app", helpers)
if len(diagnostics) != 0 {
t.Fatalf("iterator diagnostics = %#v", diagnostics)
}
if candidate, ok, err := iterator.next(); err != nil || !ok || candidate != "/selected.yml" {
t.Fatalf("first candidate = %q, %v, %v", candidate, ok, err)
}
if configCalls != 0 {
t.Fatalf("later user directory resolved before consumption: %d calls", configCalls)
}
emptyOnly := SearchDirectory("should-not-resolve", DirectorySearchOptions{Filenames: []FilenameGroup{}})
emptyCandidates := collectSearchCandidates(t, emptyOnly, "", helpers)
if len(emptyCandidates) != 0 || configCalls != 0 {
t.Fatalf("empty directory candidates = %#v, helper calls = %d", emptyCandidates, configCalls)
}
continued := collectSearchCandidates(t, Searches(emptyOnly, Paths("later.yml")), "", helpers)
assertCandidatePaths(t, continued, []string{filepath.Join("/controlled/work", "later.yml")})
if configCalls != 0 {
t.Fatalf("empty directory base was resolved: %d calls", configCalls)
}
}
func TestSearchCandidateBaseFailuresAndPlatformApplicability(t *testing.T) {
t.Parallel()
baseFailure := errors.New("controlled base failure")
helpers := discoveryDirectoryHelpers{
userConfigDir: func() (string, error) { return "", baseFailure },
workingDir: func() (string, error) { return "/controlled/work", nil },
unixSupported: func() bool { return false },
}
iterator, diagnostics := newSearchCandidateIterator(UserConfigDirectory(), "app", helpers)
if len(diagnostics) != 0 {
t.Fatalf("user preset validation = %#v", diagnostics)
}
if _, ok, err := iterator.next(); ok || !errors.Is(err, baseFailure) {
t.Fatalf("unavailable user base = ok %v, err %v", ok, err)
}
iterator, diagnostics = newSearchCandidateIterator(UnixSystemDirectories(), "app", helpers)
if len(diagnostics) != 0 {
t.Fatalf("Unix preset validation = %#v", diagnostics)
}
if _, ok, err := iterator.next(); ok || !errors.Is(err, errUnixSystemPresetUnavailable) {
t.Fatalf("unsupported Unix preset = ok %v, err %v", ok, err)
}
iterator, diagnostics = newSearchCandidateIterator(Searches(Paths("/earlier.yml"), UnixSystemDirectories()), "app", helpers)
if len(diagnostics) != 0 {
t.Fatalf("composed Unix preset validation = %#v", diagnostics)
}
if candidate, ok, err := iterator.next(); err != nil || !ok || candidate != "/earlier.yml" {
t.Fatalf("earlier candidate = %q, %v, %v", candidate, ok, err)
}
}
func TestRelativeAndShellLikeSearchPathsRemainLiteral(t *testing.T) {
t.Parallel()
root := t.TempDir()
helpers := discoveryDirectoryHelpers{workingDir: func() (string, error) { return root, nil }}
search := Searches(Paths("~/literal.yml", "$CONFIG/*.yaml"), SearchDirectory("./settings", DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("name with spaces.yml")}}))
candidates := collectSearchCandidates(t, search, "", helpers)
assertCandidatePaths(t, candidates, []string{
filepath.Join(root, "~/literal.yml"),
filepath.Join(root, "$CONFIG/*.yaml"),
filepath.Join(root, "settings", "name with spaces.yml"),
})
}
func collectSearchCandidates(t *testing.T, search Search, application string, helpers discoveryDirectoryHelpers) []string {
t.Helper()
iterator, diagnostics := newSearchCandidateIterator(search, application, helpers)
if len(diagnostics) != 0 {
t.Fatalf("search validation = %#v", diagnostics)
}
var candidates []string
for {
candidate, ok, err := iterator.next()
if err != nil {
t.Fatalf("expand candidate: %v", err)
}
if !ok {
return candidates
}
candidates = append(candidates, candidate)
}
}
func assertCandidatePaths(t *testing.T, got, want []string) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("candidate count = %d, want %d: %#v", len(got), len(want), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("candidate %d = %q, want %q", i, got[i], want[i])
}
}
}
func TestResolveSearchSelectsFirstExistingCandidate(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := filepath.Join(root, "first.yml")
second := filepath.Join(root, "second.yml")
writeDiscoveryFile(t, first, "value: first\n")
writeDiscoveryFile(t, second, "value: second\n")
selected, diagnostics := resolveSearchSource(Source{}, ptrSearch(Paths(filepath.Join(root, "missing.yml"), first, second)), "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.kind != sourceFile || selected.path != first {
t.Fatalf("selection = %#v, diagnostics %#v", selected, diagnostics)
}
read, diagnostics := readSource(selected, "source-1", defaultSourceLimits())
if len(diagnostics) != 0 || string(read.data) != "value: first\n" || read.info.Path != canonicalTestPath(t, first) {
t.Fatalf("selected source read = %#v, diagnostics %#v", read, diagnostics)
}
}
func TestResolveSearchNamedExtensionsAndNonregularSelection(t *testing.T) {
t.Parallel()
root := t.TempDir()
writeDiscoveryFile(t, filepath.Join(root, "app.yml"), "value: yml\n")
writeDiscoveryFile(t, filepath.Join(root, "app.yaml"), "value: yaml\n")
selected, diagnostics := resolveSearchSource(Source{}, ptrSearch(SearchDirectory(root)), "app", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.path != filepath.Join(root, "app.yml") {
t.Fatalf("named extension selection = %#v, %#v", selected, diagnostics)
}
directory := filepath.Join(root, "subdirectory")
if err := os.Mkdir(directory, 0o700); err != nil {
t.Fatal(err)
}
good := filepath.Join(root, "good.yml")
writeDiscoveryFile(t, good, "value: later\n")
selected, diagnostics = resolveSearchSource(Source{}, ptrSearch(Paths(directory, good)), "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.path != directory {
t.Fatalf("directory candidate selection = %#v, %#v", selected, diagnostics)
}
_, diagnostics = readSource(selected, "source-1", defaultSourceLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceNotRegular {
t.Fatalf("selected directory read diagnostics = %#v", diagnostics)
}
}
func TestResolveSearchContainedSymlinkAndNoFallbackAfterReadFailure(t *testing.T) {
t.Parallel()
root := t.TempDir()
target := filepath.Join(root, "actual.yml")
link := filepath.Join(root, "linked.yml")
writeDiscoveryFile(t, target, "value: linked\n")
if err := os.Symlink(target, link); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
writeDiscoveryFile(t, filepath.Join(root, "later.yml"), "value: later\n")
selected, diagnostics := resolveSearchSource(Source{}, ptrSearch(Paths(link, filepath.Join(root, "later.yml"))), "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.path != link {
t.Fatalf("symlink selection = %#v, %#v", selected, diagnostics)
}
read, diagnostics := readSource(selected, "source-1", defaultSourceLimits())
if len(diagnostics) != 0 || string(read.data) != "value: linked\n" || read.info.Path != canonicalTestPath(t, target) {
t.Fatalf("symlink read = %#v, %#v", read, diagnostics)
}
if err := os.Remove(target); err != nil {
t.Fatal(err)
}
_, diagnostics = readSource(selected, "source-1", defaultSourceLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceMissing {
t.Fatalf("selected source read failure = %#v", diagnostics)
}
}
func TestResolveSearchRequiredAndOptionalExhaustion(t *testing.T) {
t.Parallel()
optional, diagnostics := resolveSearchSource(Source{}, ptrSearch(Search{}), "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || optional.kind != sourceNone {
t.Fatalf("optional empty search = %#v, %#v", optional, diagnostics)
}
_, diagnostics = resolveSearchSource(Source{}, ptrSearch(Search{}), "", true, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceMissing || diagnostics[0].Primary != nil || !strings.Contains(diagnostics[0].Message, "search") {
t.Fatalf("required empty search = %#v", diagnostics)
}
root := t.TempDir()
missing := filepath.Join(root, "missing.yml")
optional, diagnostics = resolveSearchSource(Source{}, ptrSearch(Paths(missing)), "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || optional.kind != sourceNone {
t.Fatalf("optional missing search = %#v, %#v", optional, diagnostics)
}
_, diagnostics = resolveSearchSource(Source{}, ptrSearch(Paths(missing)), "", true, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourceMissing {
t.Fatalf("required missing search = %#v", diagnostics)
}
}
func TestResolveSearchLeavesNonSearchSourcesUntouched(t *testing.T) {
t.Parallel()
source := Bytes("memory", []byte("value: unchanged\n"))
resolved, diagnostics := resolveSearchSource(source, nil, "", false, discoveryDirectoryHelpers{}, discoveryHostProbe{})
if len(diagnostics) != 0 || resolved.kind != sourceBytes || string(resolved.data) != "value: unchanged\n" {
t.Fatalf("non-search source = %#v, diagnostics = %#v", resolved, diagnostics)
}
}
func TestResolveSearchStopsOnReachedProbeError(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := filepath.Join(root, "first.yml")
second := filepath.Join(root, "second.yml")
writeDiscoveryFile(t, first, "value: first\n")
writeDiscoveryFile(t, second, "value: second\n")
statCalls := 0
probe := discoveryHostProbe{stat: func(path string) (fs.FileInfo, error) {
statCalls++
return nil, fs.ErrPermission
}}
selected, info, diagnostics := resolveSearchInput(SearchConfig("", Paths(first, second)), "source-3", discoveryDirectoryHelpers{}, probe)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSourcePermission || statCalls != 1 {
t.Fatalf("probe failure = %#v, stat calls = %d", diagnostics, statCalls)
}
if selected.kind != sourceFile || info == nil || info.Path != canonicalTestPath(t, first) || info.Label != first || info.Kind != SourceKindFile || info.ID != "source-3" {
t.Fatalf("failed candidate evidence = %#v, %#v", selected, info)
}
if diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != info.ID {
t.Fatalf("failed candidate diagnostic does not identify its file: %#v", diagnostics)
}
}
func TestResolveSearchLeavesLaterDirectoryUnresolvedAfterSelection(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := filepath.Join(root, "selected.yml")
writeDiscoveryFile(t, first, "value: selected\n")
baseCalls := 0
helpers := discoveryDirectoryHelpers{userConfigDir: func() (string, error) {
baseCalls++
return "", errors.New("later base should not be reached")
}}
search := ptrSearch(Searches(Paths(first), UserConfigDirectory()))
selected, diagnostics := resolveSearchSource(Source{}, search, "app", false, helpers, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.path != first || baseCalls != 0 {
t.Fatalf("selected = %#v, diagnostics = %#v, later base calls = %d", selected, diagnostics, baseCalls)
}
}
func TestResolveSearchTreatsShellLikeNamesLiterally(t *testing.T) {
t.Parallel()
root := t.TempDir()
paths := []string{"~/literal.yml", "$CONFIG/literal.yml", "*.yaml"}
for _, name := range paths {
writeDiscoveryFile(t, filepath.Join(root, name), "value: literal\n")
}
helper := discoveryDirectoryHelpers{workingDir: func() (string, error) { return root, nil }}
for _, name := range paths {
selected, diagnostics := resolveSearchSource(Source{}, ptrSearch(Paths(name)), "", false, helper, discoveryHostProbe{})
if len(diagnostics) != 0 || selected.path != filepath.Join(root, name) {
t.Fatalf("literal %q selection = %#v, %#v", name, selected, diagnostics)
}
}
}
func ptrSearch(search Search) *Search { return &search }
func writeDiscoveryFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,91 @@
//go:build windows
package engine
import (
"errors"
"path/filepath"
"strings"
"testing"
)
func TestWindowsSearchRejectsReservedComponentsBeforeAcquisition(t *testing.T) {
t.Parallel()
root := t.TempDir()
first := writeImportFile(t, root, "first.yml", "value: first\n")
for _, name := range []string{"CONIN$", "CONOUT$"} {
for _, source := range []Source{
SearchConfig(name, Searches(Paths(first), WorkingDirectory())),
SearchConfig("", Searches(Paths(first), SearchDirectory(root, DirectorySearchOptions{
Filenames: []FilenameGroup{CustomFilenames(name)},
}))),
} {
result, err := testLoad[discoveryLoadConfig](source, Options[discoveryLoadConfig]{})
var failure *LoadError
if result != nil || !errors.As(err, &failure) || len(failure.Diagnostics()) != 1 || failure.Diagnostics()[0].Code != CodeOptionsInvalid {
t.Fatalf("reserved component %q accepted: %#v, %v", name, result, err)
}
if len(failure.Sources()) != 0 {
t.Fatalf("reserved component %q was checked after acquisition: %#v", name, failure.Sources())
}
}
}
}
func TestWindowsSearchPathsMatchHostAbsoluteResolution(t *testing.T) {
root := t.TempDir()
file := writeImportFile(t, root, "settings/config.yml", "value: selected\n")
working := t.TempDir()
t.Chdir(working)
relative, err := filepath.Rel(working, file)
if err != nil {
t.Fatal(err)
}
volume := filepath.VolumeName(file)
rooted := strings.TrimPrefix(file, volume)
for _, entry := range []struct {
name string
path string
}{
{"relative", relative},
{"drive root relative", rooted},
{"slash root relative", filepath.ToSlash(rooted)},
{"drive relative", volume + relative},
{"absolute", file},
} {
t.Run(entry.name, func(t *testing.T) {
want, err := filepath.Abs(entry.path)
if err != nil {
t.Fatal(err)
}
explicit, err := testLoad[discoveryLoadConfig](File(entry.path))
if err != nil {
t.Fatal(err)
}
for _, search := range []Search{
Paths(entry.path),
SearchDirectory(filepath.Dir(entry.path), DirectorySearchOptions{
Filenames: []FilenameGroup{CustomFilenames(filepath.Base(entry.path))},
}),
} {
iterator, diagnostics := newSearchCandidateIterator(search, "", discoveryDirectoryHelpers{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
candidate, ok, err := iterator.next()
if err != nil || !ok || candidate != want {
t.Fatalf("candidate = %q, %v, %v; want %q", candidate, ok, err, want)
}
result, err := testLoad[discoveryLoadConfig](SearchConfig("", search))
if err != nil || result.Config.Value != explicit.Config.Value {
t.Fatalf("searched path = %#v, %v; explicit value %q", result, err, explicit.Config.Value)
}
sources := result.Sources()
if len(sources) != 1 || sources[0].Path != explicit.Sources()[0].Path {
t.Fatalf("searched source = %#v; explicit source %#v", sources, explicit.Sources())
}
}
})
}
}

View File

@@ -76,20 +76,55 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) {
// Intermediate input replacements do not permanently suppress defaults:
// decoding applies them to the final input tree. Show that reconciliation
// explicitly, including canonical scalar conversions, before domain changes.
// explicitly before typed CLI contributions and domain resolution.
decodedInput := r.preCLI
decodedOrigins := r.preCLIOrigins
replayed, _ := documentAtPath(working, path)
decoded, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup)
decoded, _ := documentAtPathWithLookup(decodedInput, path, r.immutableLookup)
if !configurationValuesEqual(replayed, decoded) {
origin := cloneOrigin(decodedOrigins[path.String()])
origin.Operation = ChangeDecoding
explanation.History = append(explanation.History, ValueChange{
Before: valueSnapshot(working, path, r.secrets),
After: valueSnapshotWithLookup(decodedInput, path, r.secrets, r.immutableLookup),
Origin: origin,
Reasons: []string{"applied retained defaults and canonical typed decoding"},
})
}
working = document.Clone(decodedInput)
for _, application := range r.postDecodeApps {
operationPath, relevant := applicationOperationPath(application, path, r.immutableLookup)
var before ValueSnapshot
if relevant {
before = valueSnapshot(working, path, r.secrets)
}
applyHistoryApplication(&working, application, replayIndex)
if !relevant {
continue
}
after := valueSnapshot(working, path, r.secrets)
if !before.Present && !after.Present {
continue
}
explanation.History = append(explanation.History, ValueChange{
Before: before,
After: after,
Origin: historyApplicationOrigin(application, operationPath, r.immutableLookup),
})
}
replayed, _ = documentAtPath(working, path)
effectiveInput, _ := documentAtPathWithLookup(r.input, path, r.immutableLookup)
if !configurationValuesEqual(replayed, effectiveInput) {
origin := cloneOrigin(r.inputOrigins[path.String()])
origin.Operation = ChangeDecoding
explanation.History = append(explanation.History, ValueChange{
Before: valueSnapshot(working, path, r.secrets),
After: valueSnapshotWithLookup(r.input, path, r.secrets, r.immutableLookup),
Origin: origin,
Reasons: []string{"applied retained defaults and canonical typed decoding"},
Reasons: []string{"applied the typed CLI assignment"},
})
}
working = document.Clone(r.input)
for _, snapshot := range r.resolutionReplay {
relevant := pathsIntersect(snapshot.change.Path, path)

View File

@@ -351,6 +351,10 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) {
opts.Overlay = "p"
}
result, err := testLoad(File(main), opts)
if cli && base == "replaced-text" {
requireSecretLoadCode(t, err, CodeDecodeType)
return
}
if err != nil {
t.Fatal(err)
}
@@ -360,6 +364,18 @@ func TestExplanationReconcilesDefaultsAfterInputReplacement(t *testing.T) {
t.Fatal("missing explanation")
}
last := explanation.History[len(explanation.History)-1]
if cli && path.String() == "/m" {
if last.After != explanation.Effective || last.Origin.Layer != LayerCLI || last.Origin.Operation != "" {
t.Fatalf("CLI assignment was not the final history event: %#v", explanation)
}
continue
}
if cli && path.String() == "/m/a" {
if explanation.Effective.Present {
t.Fatalf("CLI map override retained descendants discarded by source null: %#v", explanation)
}
continue
}
if last.After != explanation.Effective || last.Origin.Operation != ChangeDecoding {
t.Fatalf("history does not reconcile final value: %#v", explanation)
}

View File

@@ -9,6 +9,9 @@ import (
)
func validateFilesystemContext(source Source, context *FilesystemContext, main parsedSource) *Diagnostic {
if source.kind == sourceNone && context.FS == nil {
return validateAbsentHostFilesystemContext(context)
}
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil {
@@ -33,10 +36,57 @@ func validateFilesystemContext(source Source, context *FilesystemContext, main p
return nil
}
func validateAbsentHostFilesystemContext(context *FilesystemContext) *Diagnostic {
rootPath, err := canonicalExistingPath(context.Root)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "configuration root could not be canonicalized", nil)
return &value
}
basePath := rootPath
if context.BaseDir != "" {
basePath, err = canonicalExistingPath(context.BaseDir)
if err != nil {
value := importDiagnostic(CodeImportInvalid, "import base directory could not be canonicalized", nil)
return &value
}
}
if !containedPath(rootPath, basePath) {
value := importDiagnostic(CodeImportOutsideRoot, "import base must stay inside the configuration root", nil)
return &value
}
for _, candidate := range []string{rootPath, basePath} {
info, err := os.Stat(candidate)
if err != nil || !info.IsDir() {
value := importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)
return &value
}
}
root, err := os.OpenRoot(rootPath)
if err != nil {
value := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
value.Primary = nil
return &value
}
if err := root.Close(); err != nil {
value := importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)
return &value
}
return nil
}
func filesystemOptionDiagnostics(source Source, context *FilesystemContext) []Diagnostic {
switch source.kind {
case sourceNone:
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource cannot use filesystem context")}
return absentFilesystemOptionDiagnostics(context)
case sourceSearch:
if context.FS != nil {
return absentFilesystemOptionDiagnostics(context)
}
// A selected host file can derive its root; absence requires an
// explicit root. Check that compatibility only after selection.
if context.BaseDir != "" && !filepath.IsAbs(context.BaseDir) {
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "filesystem base directory must be an absolute host path")}
}
case sourceFile:
if context.FS != nil {
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "host file sources cannot use a native filesystem context")}
@@ -69,6 +119,17 @@ func filesystemOptionDiagnostics(source Source, context *FilesystemContext) []Di
return nil
}
func absentFilesystemOptionDiagnostics(context *FilesystemContext) []Diagnostic {
if context.FS != nil {
if context.FS.namespace == nil || nilFilesystem(context.FS.namespace.filesystem) || context.BaseDir == "" || context.Root == "" || !fs.ValidPath(context.BaseDir) || !fs.ValidPath(context.Root) {
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "native filesystem context requires a namespace, base, and root directory")}
}
} else if context.Root == "" || !filepath.IsAbs(context.Root) || context.BaseDir != "" && !filepath.IsAbs(context.BaseDir) {
return []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "absent-input filesystem root and base directories must be absolute host paths")}
}
return nil
}
func nativeContext(source Source, context *FilesystemContext) bool {
return source.kind == sourceFilesystem || context != nil && context.FS != nil
}

View File

@@ -0,0 +1,51 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func validateFinalSequenceIdentities(root *document.Node, policy sequencePolicy) []Diagnostic {
if root == nil || policy.selectorTrie == nil {
return nil
}
var diagnostics []Diagnostic
var visit func(*document.Node, Path, []*sequenceSelectorTrie)
visit = func(node *document.Node, path Path, states []*sequenceSelectorTrie) {
if node == nil {
return
}
if node.Kind == document.Sequence {
var fields []string
for _, state := range states {
if state.terminal {
fields = state.fields
break
}
}
if fields != nil {
_, issues := validateNormalizedSequenceIdentityFrom(node, path, fields, sequenceIdentityDiagnosticCodes{
invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict,
}, 0, sequencePayloadRecord)
diagnostics = append(diagnostics, issues...)
}
return
}
if node.Kind != document.Mapping {
return
}
for _, entry := range node.Mapping {
next := make([]*sequenceSelectorTrie, 0, len(states)*2)
for _, state := range states {
if literal := state.literals[entry.Key]; literal != nil {
next = append(next, literal)
}
if state.wildcard != nil {
next = append(next, state.wildcard)
}
}
if len(next) != 0 {
visit(entry.Value, path.Key(entry.Key), next)
}
}
}
visit(root, Path{}, []*sequenceSelectorTrie{policy.selectorTrie})
return canonicalDiagnostics(diagnostics)
}

View File

@@ -268,6 +268,48 @@ func TestFilesystemContextIsValidatedWithoutImportsOrActiveOverlay(t *testing.T)
}
}
func TestNoSourceImportsAndFilesystemContext(t *testing.T) {
t.Parallel()
directory := t.TempDir()
for _, imports := range []bool{false, true} {
name := "imports disabled"
if imports {
name = "imports enabled"
}
t.Run(name, func(t *testing.T) {
result, err := testLoad[importTestConfig](NoSource(), Options[importTestConfig]{
Imports: imports, Filesystem: &FilesystemContext{Root: directory},
})
if err != nil {
t.Fatal(err)
}
if sources := result.Sources(); len(sources) != 0 {
t.Fatalf("NoSource sources = %#v", sources)
}
if _, ok := result.Metadata().Lookup(Keys("items", "root")); ok {
t.Fatal("NoSource contributed explicit input")
}
})
}
outside := t.TempDir()
for _, test := range []struct {
name string
context FilesystemContext
code Code
}{
{name: "missing root", context: FilesystemContext{BaseDir: directory}, code: CodeOptionsInvalid},
{name: "relative root", context: FilesystemContext{Root: "."}, code: CodeOptionsInvalid},
{name: "missing root directory", context: FilesystemContext{Root: filepath.Join(directory, "missing")}, code: CodeImportInvalid},
{name: "base outside root", context: FilesystemContext{BaseDir: outside, Root: directory}, code: CodeImportOutsideRoot},
} {
t.Run(test.name, func(t *testing.T) {
requireLoadCode(t, NoSource(), Options[importTestConfig]{Filesystem: &test.context}, test.code)
})
}
}
func writeGlobRoot(t *testing.T, directory, pattern string, required bool) string {
t.Helper()
requiredLine := ""

View File

@@ -42,15 +42,15 @@ func (c *retainedNodeCounter) add(root *document.Node) bool {
// History applications borrow immutable policy-validated input trees. Metadata
// records answer presence and origin queries; replay reconstructs value changes.
// Replay reconciles with canonical decoding before domain resolution so both
// contracts agree on the effective result without retaining a tree per field.
// Replay reconciles with canonical decoding, applies supplied CLI contributions,
// then records domain resolution without retaining a tree per field.
type historyApplication struct {
layer Layer
path Path
value *document.Node
origin Origin
sequences []sequenceHistoryContribution
sequencesByPath map[string]sequenceHistoryContribution
sequencesByPath map[string]sequenceHistoryContributionIndex
}
func cloneHistoryApplications(values []historyApplication) []historyApplication {
@@ -92,7 +92,14 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef
for index, child := range node.Sequence {
effectiveIndex := index
if projected {
effectiveIndex += contribution.offset
if contribution.mode == sequenceHistoryMapped {
effectiveIndex = contribution.outputForInput(index)
if effectiveIndex < 0 {
continue
}
} else {
effectiveIndex += contribution.offset
}
}
walk(child, local.Index(index), effective.Index(effectiveIndex))
}
@@ -112,7 +119,18 @@ func historyApplicationLocalPath(application historyApplication, effective Path)
for _, segment := range relative.segments {
if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found {
index, valid := schemaIndex(segment)
if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length {
if !valid || segment.kind == pathSegmentKey {
return Path{}, false
}
if contribution.mode == sequenceHistoryMapped {
inputIndex := contribution.inputForOutput(index)
if inputIndex < 0 {
return Path{}, false
}
local = local.Index(inputIndex)
continue
}
if index < contribution.offset || index >= contribution.offset+contribution.length {
return Path{}, false
}
local = local.Index(index - contribution.offset)
@@ -149,6 +167,16 @@ func applicationOperationPath(application historyApplication, query Path, lookup
return container, true
}
}
indexed, _ := sequenceHistoryAt(application.sequencesByPath, contribution.path)
if indexed.mode == sequenceHistoryMapped && len(query.segments) > len(container.segments) {
outputIndex, valid := schemaIndex(query.segments[len(container.segments)])
_, replaced := indexed.replacedOutputs[outputIndex]
if valid && replaced {
if _, present := documentAtPathWithLookup(application.value, local, lookup); !ok || !present {
return container.Index(outputIndex), true
}
}
}
if !ok {
continue
}
@@ -206,14 +234,14 @@ func applyHistoryApplication(root **document.Node, application historyApplicatio
mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath)
return
}
mergeHistoryAtPath(root, application.path, application.value, index)
mergeHistoryAtPath(root, application.path, application.value, index, application.sequencesByPath)
}
func mergeHistoryDocument(target **document.Node, incoming *document.Node, index *mappingIndex) {
mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil)
}
func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) {
func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContributionIndex) {
if incoming == nil {
return
}
@@ -238,23 +266,53 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node,
}
}
func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
func sequenceHistoryAt(contributions map[string]sequenceHistoryContributionIndex, path Path) (sequenceHistoryContributionIndex, bool) {
contribution, found := contributions[path.String()]
return contribution, found
}
func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution {
type sequenceHistoryContributionIndex struct {
sequenceHistoryContribution
projection sequenceItemProjectionIndex
replacedOutputs map[int]struct{}
}
func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContributionIndex {
if len(values) == 0 {
return nil
}
index := make(map[string]sequenceHistoryContribution, len(values))
index := make(map[string]sequenceHistoryContributionIndex, len(values))
for _, value := range values {
index[value.path.String()] = value
contribution := sequenceHistoryContributionIndex{sequenceHistoryContribution: value}
if value.mode == sequenceHistoryMapped {
contribution.projection = newSequenceItemProjectionIndex(value.items)
contribution.replacedOutputs = make(map[int]struct{}, len(value.replacements))
for _, replacement := range value.replacements {
contribution.replacedOutputs[replacement.outputIndex] = struct{}{}
}
}
index[value.path.String()] = contribution
}
return index
}
func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) {
func (contribution sequenceHistoryContributionIndex) outputForInput(inputIndex int) int {
outputIndex, ok := contribution.projection.outputForInput(inputIndex)
if !ok {
return -1
}
return outputIndex
}
func (contribution sequenceHistoryContributionIndex) inputForOutput(outputIndex int) int {
inputIndex, ok := contribution.projection.inputForOutput(outputIndex)
if !ok {
return -1
}
return inputIndex
}
func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContributionIndex) {
if incoming == nil || incoming.Kind != document.Sequence {
return
}
@@ -262,6 +320,18 @@ func applySequenceHistory(target **document.Node, incoming *document.Node, contr
*target = document.Clone(incoming)
return
}
if contribution.mode == sequenceHistoryMapped {
for _, projection := range contribution.items {
if projection.inputIndex < 0 || projection.inputIndex >= len(incoming.Sequence) || projection.outputIndex < 0 {
continue
}
for len((*target).Sequence) <= projection.outputIndex {
(*target).Sequence = append((*target).Sequence, nil)
}
(*target).Sequence[projection.outputIndex] = document.Clone(incoming.Sequence[projection.inputIndex])
}
return
}
for _, item := range incoming.Sequence {
(*target).Sequence = append((*target).Sequence, document.Clone(item))
}
@@ -275,13 +345,15 @@ func cloneSequenceHistoryContributions(values []sequenceHistoryContribution) []s
for index, value := range values {
cloned[index] = value
cloned[index].path = value.path.clone()
cloned[index].items = append([]sequenceItemProjection(nil), value.items...)
cloned[index].replacements = append([]sequenceItemReplacement(nil), value.replacements...)
}
return cloned
}
func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node, index *mappingIndex) {
func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node, index *mappingIndex, sequences map[string]sequenceHistoryContributionIndex) {
if root == nil || *root == nil || len(path.segments) == 0 {
mergeHistoryDocument(root, incoming, index)
mergeHistoryDocumentAtPath(root, incoming, Path{}, index, sequences)
return
}
current := *root
@@ -292,7 +364,7 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
return
}
if segmentIndex == len(path.segments)-1 {
mergeHistoryDocument(&current.Sequence[position], incoming, index)
mergeHistoryDocumentAtPath(&current.Sequence[position], incoming, Path{}, index, sequences)
return
}
current = current.Sequence[position]
@@ -306,7 +378,7 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
if position < 0 {
index.append(current, document.Entry{Key: segment.key, Value: document.Clone(incoming)})
} else {
mergeHistoryDocument(&current.Mapping[position].Value, incoming, index)
mergeHistoryDocumentAtPath(&current.Mapping[position].Value, incoming, Path{}, index, sequences)
}
return
}

View File

@@ -10,7 +10,10 @@ import (
)
func TestSequenceHistoryReplaysAppendAtAssignedIndexes(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id", "/other": "id"}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("other")}, identityFields: []string{"id"}},
})
first := parseCompositionDocument(t, "source-1", "records: [{id: a, value: first}]\nother: [{id: x}]\n")
second := parseCompositionDocument(t, "source-2", "records: [{id: b, value: second}]\nother: [{id: y}]\n")
composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
@@ -75,8 +78,76 @@ func TestSequenceHistoryReplaysAppendAtAssignedIndexes(t *testing.T) {
}
}
func TestTierSequenceHistoryReplaysSelectedOperation(t *testing.T) {
t.Parallel()
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"},
}})
for _, test := range []struct {
name string
mode sequenceMode
}{{"append", sequenceModeAppend}, {"upsert", sequenceModeUpsert}, {"replace", sequenceModeReplace}} {
t.Run(test.name, func(t *testing.T) {
mode := test.mode
lower, diagnostics := composeDocumentsWithPolicy(policy, parseCompositionDocument(t, "lower", "records: [{id: a, removed: old}, {id: b, value: old, removed: old}]\n"))
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
secondID := "d"
if mode == sequenceModeUpsert {
secondID = "b"
}
declarations := []sequenceDeclarationSpec{{destination: Keys("records"), modePresent: true, mode: mode}}
higher, diagnostics := composeDocumentsWithCollectionPolicy(policy, lower.collectionPolicies, declarations,
parseCompositionDocument(t, "first-peer", "records: [{id: c, value: first}]\n"),
parseCompositionDocument(t, "second-peer", "records: [{id: "+secondID+", value: second}]\n"),
)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
assembly := newAssemblyAccumulator(lower)
if diagnostics := assembly.applyOverlayWithCollectionPolicy(higher.root, higher.origins, higher.collectionPolicies, higher.applications, LayerBase, policy, declarations); len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
if len(assembly.current.applications) != 3 {
t.Fatalf("source applications = %d, want all three sources", len(assembly.current.applications))
}
if mode == sequenceModeUpsert {
query := Keys("records").Index(1).Key("value")
if _, relevant := applicationOperationPath(assembly.current.applications[1], query, nil); relevant {
t.Fatal("first peer was attributed to another peer's upserted record")
}
origin := historyApplicationOrigin(assembly.current.applications[2], query, nil)
if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "second-peer" {
t.Fatalf("upsert history source = %#v", origin)
}
}
working := parseCompositionDocument(t, "initial", "{}\n")
index := new(mappingIndex)
for _, application := range assembly.current.applications {
applyHistoryApplication(&working, application, index)
}
if !configurationValuesEqual(working, assembly.current.root) {
t.Fatal("tier history differs from structural assembly before decoding")
}
overlay := parseCompositionDocument(t, "overlay", "records: [{id: c, value: overlay}]\n")
declarations[0].mode = sequenceModeUpsert
if diagnostics := assembly.applyOverlayWithCollectionPolicy(overlay, nil, collectionPolicyScope{}, nil, LayerOverlay, policy, declarations); len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
applyHistoryApplication(&working, assembly.current.applications[3], index)
if !configurationValuesEqual(working, assembly.current.root) {
t.Fatal("subsequent overlay history differs from structural assembly")
}
})
}
}
func TestSequenceHistoryPreservesTierReplacementAndEmptyLists(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id", "/independent": "id"}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{
{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}},
{selector: sequenceIdentitySelector{literalSequenceSelector("independent")}, identityFields: []string{"id"}},
})
first, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "underlay-1a", "records: [{id: shared}]\nindependent: [{id: x}]\n"),
parseCompositionDocument(t, "underlay-1b", "records: [{id: lower}]\n"),
@@ -145,10 +216,53 @@ func TestHistoryMergeSupportsSemanticIndexedPrefixes(t *testing.T) {
if !present || value.Text != "new" {
t.Fatalf("indexed helper replay = %#v, %v", value, present)
}
root = parseCompositionDocument(t, "base", "records: [{id: one, helper: {items: [{id: a}]}}]\n")
application.value = parseCompositionDocument(t, "helper", "items: [{id: b}]\n")
application.sequences = []sequenceHistoryContribution{{path: Keys("items"), mode: sequenceHistoryAppend, offset: 1, length: 1}}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
applyHistoryApplication(&root, application, new(mappingIndex))
items, present := documentAtPath(root, Keys("records").Index(0).Key("helper").Key("items"))
if !present || len(items.Sequence) != 2 {
t.Fatalf("indexed helper lost its local sequence operation: %#v", items)
}
}
func TestLayeredUpsertExplanationFollowsStructuralOrder(t *testing.T) {
t.Parallel()
type config struct {
Records []map[string]any `yaml:"records"`
}
loaded, err := testGroupedLoad(loadRequest[config]{
main: loadInput{source: Bytes("main", []byte("configkit:\n sequences: [{destination: [records], key: id}]\nrecords: [{id: b, value: new-b}, {id: c, value: new-c}, {id: a, value: new-a}]\n"))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("records: [{id: a, value: old-a}, {id: b, value: old-b, removed: old}]\n"))}}}},
})
if err != nil {
t.Fatal(err)
}
explanation, ok := loaded.Explain(Keys("records"))
if !ok || len(explanation.History) == 0 {
t.Fatal("missing collection history")
}
for _, change := range explanation.History {
if change.Origin.Operation == ChangeDecoding {
t.Fatal("structural upsert was misreported as a decoding reconciliation")
}
}
if last := explanation.History[len(explanation.History)-1]; last.After.YAML != explanation.Effective.YAML || last.Origin.Layer != LayerBase {
t.Fatalf("main contribution did not produce the effective collection: %#v", last)
}
removed, ok := loaded.Explain(Keys("records").Index(1).Key("removed"))
if !ok || len(removed.History) == 0 || removed.Effective.Present {
t.Fatalf("missing displaced field history: %#v, %v", removed, ok)
}
if last := removed.History[len(removed.History)-1]; last.After.Present || last.Origin.Layer != LayerBase || last.Origin.Operation == ChangeDecoding {
t.Fatalf("upsert removal attribution = %#v", last)
}
}
func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
parseCompositionDocument(t, "second", "records: [{id: second, token: private}]"),
@@ -165,7 +279,7 @@ func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T
inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: composed.root, final: composed.root, applications: composed.applications,
input: composed.root, preCLI: composed.root, final: composed.root, applications: composed.applications,
}
inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
return reflect.DeepEqual(candidate, path)
@@ -193,7 +307,7 @@ func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T
}
func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}}
policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
composed, diagnostics := composeDocumentsWithPolicy(policy,
parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
parseCompositionDocument(t, "second", "records: [{id: second, token: PRIVATE-RECORD-CANARY}]"),
@@ -212,7 +326,7 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
finalizeMetadataPaths(&metadata, final, applications)
inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: final, final: final, applications: applications,
input: final, preCLI: final, final: final, applications: applications,
}
inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
return reflect.DeepEqual(candidate, path)
@@ -230,6 +344,241 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
}
}
func TestMappedUpsertProjectsSensitiveReplacementEvidence(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
- id: middle
- id: last
token: MAPPED-SECRET-CANARY
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
token: visible
- id: first
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher,
sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
secretPath := Keys("records").Index(2).Key("token")
secretNode, present := documentAtPath(lowerRoot, Keys("records").Index(2).Key("token"))
if !present {
t.Fatal("sensitive source token is missing")
}
secretNode.Sensitive = true
application := historyApplication{
layer: LayerOverlay, value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
applications := []historyApplication{{value: lowerRoot}, application}
replayed := &document.Node{Kind: document.Mapping}
for _, current := range applications {
applyHistoryApplication(&replayed, current, new(mappingIndex))
}
replayedRecords, _ := documentAtPath(replayed, Keys("records"))
if !reflect.DeepEqual(replayedRecords.Sequence, combined.value.Sequence) {
t.Fatalf("replayed value = %#v, want combination = %#v", replayedRecords.Sequence, combined.value.Sequence)
}
finalRoot := document.Clone(lowerRoot)
finalRecords, _ := documentAtPath(finalRoot, Keys("records"))
finalRecords.Sequence = document.Clone(combined.value).Sequence
var metadata Metadata
finalizeMetadataPaths(&metadata, finalRoot, applications)
inspection := &Inspection{
metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
input: finalRoot, preCLI: finalRoot, final: finalRoot, applications: applications,
}
secretRule := func(candidate Path) bool { return reflect.DeepEqual(candidate, secretPath) }
inspection.secrets = classifySecretPaths(nil, secretRule, metadata, finalRoot, applications)
explanation, ok := inspection.Explain(secretPath)
if !ok || !explanation.Effective.Redacted {
t.Fatalf("mapped replacement explanation = %#v, %v", explanation, ok)
}
var rendered bytes.Buffer
if err := inspection.WriteExplanation(&rendered, secretPath); err != nil {
t.Fatal(err)
}
if strings.Contains(rendered.String(), "MAPPED-SECRET-CANARY") || !strings.Contains(rendered.String(), "[REDACTED]") {
t.Fatalf("mapped replacement history leaked or was not redacted: %s", rendered.String())
}
}
func TestSequenceHistoryReplaysCombinationEvidence(t *testing.T) {
tests := []struct {
name string
operation sequenceMode
lower string
higher string
}{
{
name: "append",
operation: sequenceModeAppend,
lower: "records: [{id: first}, {id: last}]\n",
higher: "records: [{id: added}]\n",
},
{
name: "upsert",
operation: sequenceModeUpsert,
lower: `records:
- id: first
label: same
nested: {removed: lower}
- id: middle
- id: last
nested: {removed: lower}
`,
higher: `records:
- id: last
nested: {kept: higher}
- id: first
label: same
- id: added
`,
},
{
name: "replace",
operation: sequenceModeReplace,
lower: "records: [{id: discarded}]\n",
higher: "records: [{id: replacement}]\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", test.lower)
higherRoot := parseCompositionDocument(t, "higher", test.higher)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
fields := []string(nil)
if test.operation != sequenceModeReplace {
fields = []string{"id"}
}
combined, diagnostics := combineSequence(lower, higher, sequenceCombinationPolicy(test.operation, fields...), sequenceCombinationContext{destination: Keys("records")})
if len(diagnostics) != 0 {
t.Fatalf("combine diagnostics = %#v", diagnostics)
}
mode := sequenceHistoryMapped
if test.operation == sequenceModeReplace {
mode = sequenceHistoryReplace
}
length := 0
if mode == sequenceHistoryReplace {
length = len(combined.higherItems)
}
application := historyApplication{
layer: LayerOverlay,
value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: mode, length: length,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
replayed := document.Clone(lowerRoot)
applyHistoryApplication(&replayed, application, new(mappingIndex))
got, present := documentAtPath(replayed, Keys("records"))
if !present || !reflect.DeepEqual(got.Sequence, combined.value.Sequence) {
t.Fatalf("replayed sequence = %#v, want combination = %#v", got, combined.value.Sequence)
}
})
}
}
func TestMappedSequenceHistoryReplacesWholeRecordsAndKeepsDisplacedEvidence(t *testing.T) {
lowerRoot := parseCompositionDocument(t, "lower", `records:
- id: first
label: same
- id: middle
- id: last
nested: {removed: LOWER-CANARY}
`)
higherRoot := parseCompositionDocument(t, "higher", `records:
- id: last
nested: {kept: higher}
- id: first
label: same
- id: added
`)
lower, _ := documentAtPath(lowerRoot, Keys("records"))
higher, _ := documentAtPath(higherRoot, Keys("records"))
combined, diagnostics := combineSequence(
lower, higher,
sequenceCombinationPolicy(sequenceModeUpsert, "id"),
sequenceCombinationContext{destination: Keys("records")},
)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
application := historyApplication{
layer: LayerOverlay, value: higherRoot,
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: combined.higherItems, replacements: combined.replacements,
}},
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
clonedApplications := cloneHistoryApplications([]historyApplication{application})
clonedApplications[0].sequences[0].items[0].outputIndex = 99
clonedApplications[0].sequences[0].replacements[0].outputIndex = 99
if application.sequences[0].items[0].outputIndex == 99 || application.sequences[0].replacements[0].outputIndex == 99 {
t.Fatal("cloned mapped history aliases its source projections")
}
query := Keys("records").Index(2).Key("nested").Key("removed")
operationPath, relevant := applicationOperationPath(application, query, nil)
if !relevant || operationPath.String() != "/records/2" {
t.Fatalf("removed descendant operation = %s, %v", operationPath.String(), relevant)
}
localPath, ok := historyApplicationLocalPath(application, Keys("records").Index(2).Key("nested").Key("kept"))
if !ok || localPath.String() != "/records/0/nested/kept" {
t.Fatalf("mapped effective path localizes to %s, %v", localPath.String(), ok)
}
origin := historyApplicationOrigin(application, operationPath, nil)
if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "higher" {
t.Fatalf("replacement operation origin = %#v", origin)
}
var walkedMappedPath bool
walkHistoryApplication(application, func(local, effective Path, _ *document.Node) {
if local.String() == "/records/0/nested/kept" && effective.String() == "/records/2/nested/kept" {
walkedMappedPath = true
}
})
if !walkedMappedPath {
t.Fatal("history walk did not project a noncontiguous incoming item")
}
replayed := document.Clone(lowerRoot)
applyHistoryApplication(&replayed, application, new(mappingIndex))
sequence, _ := documentAtPath(replayed, Keys("records"))
if _, present := documentAtPath(sequence.Sequence[2], Keys("nested", "removed")); present {
t.Fatal("whole-record replacement retained an omitted descendant")
}
if value, present := documentAtPath(sequence.Sequence[2], Keys("nested", "kept")); !present || value.Text != "higher" {
t.Fatalf("replacement descendant = %#v, %v", value, present)
}
if sequence.Sequence[1].Mapping[0].Value.Text != "middle" {
t.Fatal("unchanged neighboring record moved or changed")
}
if value, present := documentAtPath(lowerRoot, Keys("records").Index(2).Key("nested").Key("removed")); !present || value.Text != "LOWER-CANARY" {
t.Fatalf("displaced lower contribution was not retained as raw history: %#v, %v", value, present)
}
if got := application.sequences[0].replacements; !reflect.DeepEqual(got, combined.replacements) || len(got) != 2 {
t.Fatalf("replacement evidence lost equal-valued or noncontiguous events: %#v", got)
}
}
func mustParsePath(t *testing.T, value string) Path {
t.Helper()
path, err := ParsePath(value)

View File

@@ -193,18 +193,26 @@ func parseImportSpecificationDirective(node *document.Node, directivePath Path)
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "configuration directive field is not recognized", entry.Value))
continue
}
if entry.Value.Kind != document.Sequence {
diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "imports must be a sequence", entry.Value))
parsed, issues := parseImportDeclarations(entry.Value, fieldPath)
specs = append(specs, parsed...)
diagnostics = append(diagnostics, issues...)
}
return specs, canonicalDiagnostics(diagnostics)
}
func parseImportDeclarations(node *document.Node, declarationPath Path) ([]importDeclarationSpec, []Diagnostic) {
if node == nil || node.Kind != document.Sequence {
return nil, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "imports must be a sequence", node)}
}
var specs []importDeclarationSpec
var diagnostics []Diagnostic
for index, declarationNode := range node.Sequence {
spec, issues := parseImportDeclarationSpec(declarationNode, declarationPath.Index(index))
if len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
continue
}
for index, declarationNode := range entry.Value.Sequence {
spec, issues := parseImportDeclarationSpec(declarationNode, fieldPath.Index(index))
if len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
continue
}
specs = append(specs, spec)
}
specs = append(specs, spec)
}
return specs, canonicalDiagnostics(diagnostics)
}

View File

@@ -46,12 +46,13 @@ type importAccounting struct {
}
type importTraversal struct {
access *filesystemAccess
accounting *importAccounting
limits importLimits
rule *VersionRule
versionState *sourceVersionState
defaults importDefaults
access *filesystemAccess
accounting *importAccounting
limits importLimits
rule *VersionRule
versionState *sourceVersionState
directiveContext rootDirectiveContext
defaults importDefaults
scope acquisitionScope
itemScope bool
@@ -166,12 +167,17 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
if versionState != nil {
defaults = snapshotImportDefaults(versionState.importDefaults)
}
directiveContext := rootDirectiveContextImportsOnly
if processing != nil && processing.directiveContext != 0 {
directiveContext = processing.directiveContext
}
state := importTraversal{
access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState,
defaults: defaults,
scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()},
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited,
directiveContext: directiveContext,
defaults: defaults,
scope: acquisitionScope{owner: mainNodeKey, destination: destination.clone()},
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited,
importGraph: newImportInclusionGraph(),
}
if mainIsItem {
@@ -201,10 +207,12 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
state.stack = append(state.stack, mainRoute)
var body *document.Node
var declarations []importDeclarationSpec
var sequenceDeclarations []sequenceDeclarationSpec
var diagnostics []Diagnostic
if processing != nil {
body = processing.payload
declarations = processing.imports
sequenceDeclarations = cloneSequenceDeclarations(processing.sequences)
} else {
body, declarations, diagnostics = splitImportDirectives(main.document)
}
@@ -213,6 +221,7 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), identity: mainNodeKey, role: importRoleMain,
scopeOwner: mainNodeKey, payload: body, document: body, processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{})
} else {
@@ -228,6 +237,7 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
source: cloneSourceInfo(mainInfo), payload: body, document: placed,
identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey,
destination: destination.clone(), processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
diagnostics = state.walkDeclarations(basePath, declarations, 0, destination)
}
@@ -235,9 +245,6 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
}
if len(diagnostics) == 0 {
state.scheduleAcquisitionScopes(&state.scope)
if !deferAssembly {
diagnostics = state.composePendingItems(&state.scope)
}
}
closeErr := access.close()
@@ -250,19 +257,19 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
if closeErr != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
}
if deferAssembly {
acquisition := state.scope
if mainIsItem {
localScope := state.scope
acquisition = acquisitionScope{
owner: mainNodeKey, destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
processing: processing, pendingItem: &localScope,
}},
}
acquisition := state.scope
if mainIsItem {
localScope := state.scope
acquisition = acquisitionScope{
owner: mainNodeKey, destination: destination.clone(),
contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations), pendingItem: &localScope,
}},
}
}
if deferAssembly {
if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
}
@@ -271,20 +278,65 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}, cloneSourceInfos(state.sources), nil
}
if mainIsItem {
localScope := state.scope
acquired := &composedDocument{
acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
}
input := assembledInput{
parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition),
sources: state.sources, versionState: versionState, accounting: accounting,
}
composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy)
return composed, cloneSourceInfos(state.sources), diagnostics
}
// composeAcquiredInputWithPolicy is the per-input adapter from retained
// acquisition scopes to the established strict-composition behavior. It uses
// the already acquired documents and graph; source handles are closed before
// this handoff.
func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
return input.composition, nil
}
acquired := input.composition
if len(acquired.acquisition) != 1 {
return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)}
}
scope := cloneAcquisitionScope(acquired.acquisition[0])
var rule *VersionRule
if input.versionState != nil {
rule = input.versionState.effectiveRule
}
accounting := input.accounting
if accounting == nil {
accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes}
}
traversal := importTraversal{
accounting: accounting,
limits: defaultImportLimits(), rule: rule, versionState: input.versionState,
importGraph: acquired.importGraph.clone(),
}
if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil {
main := scope.contributions[0]
localScope := cloneAcquisitionScope(*main.pendingItem)
if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
}
documents := make([]*document.Node, 0, len(localScope.contributions))
for _, contribution := range localScope.contributions {
documents = append(documents, contribution.document)
}
completed, failures := composeDocuments(documents...)
if len(failures) != 0 {
return nil, cloneSourceInfos(state.sources), prependCompositionDiagnosticPaths(failures, destination.Index(0))
completed, diagnostics := composeDocumentsWithCollectionPolicy(
policy, localScope.collectionPolicies,
acquiredSequenceDeclarations(localScope), documents...,
)
if len(diagnostics) != 0 {
return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
}
if versionState != nil {
protectionRule := versionState.localRule
if versionState.mode == sourceVersionModeDefault && protectionRule != nil {
prefix := destination.Index(0)
if input.versionState != nil {
protectionRule := input.versionState.localRule
if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil {
prefix := main.destination.Index(0)
relative, inside := relativeHistoryPath(prefix, protectionRule.Path)
if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) {
protectionRule = nil
@@ -298,56 +350,84 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
if rule != nil {
diagnostic.Path = rule.Path.clone()
}
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
return nil, []Diagnostic{*diagnostic}
}
}
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)}
projectNestedEvidence(completed, localScope)
_, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes)
_, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
}
placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > limits.nodes {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
if !main.placementCharged {
placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > traversal.limits.nodes {
return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)}
}
accounting.nodes += placementNodes
}
accounting.nodes += placementNodes
composed, failures := composeDocumentsWithPolicy(policy, placed)
if composed != nil {
synthetic := acquisitionScope{contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed,
identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey,
destination: destination.clone(), processing: processing, nested: completed,
}}}
synthetic.owner = mainNodeKey
projectNestedEvidence(composed, synthetic)
projectAcquiredDestinations(composed, synthetic, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes
composed.importGraph = state.importGraph.clone()
composition, diagnostics := composeDocumentsWithPolicy(policy, placed)
if len(diagnostics) != 0 {
return nil, diagnostics
}
return composed, cloneSourceInfos(state.sources), failures
synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{
source: cloneSourceInfo(main.source), payload: completed.root, document: placed,
identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner,
destination: main.destination.clone(), processing: main.processing, nested: completed,
}}}
projectNestedEvidence(composition, synthetic)
projectAcquiredDestinations(composition, synthetic, input.sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = acquired.importGraph.clone()
return composition, nil
}
documents := make([]*document.Node, 0, len(state.scope.contributions))
for _, contribution := range state.scope.contributions {
return composeAcquiredScopeWithPolicy(
&scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting,
)
}
// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group
// handoffs. Lower policy is carried separately from registered application
// constraints; current strict composition continues to use its existing rules.
func composeAcquiredScopeWithPolicy(
scope *acquisitionScope,
lower collectionPolicyScope,
policy sequencePolicy,
graph importInclusionGraph,
sources []SourceInfo,
rule *VersionRule,
versionState *sourceVersionState,
accounting *importAccounting,
) (*composedDocument, []Diagnostic) {
traversal := importTraversal{
accounting: accounting, limits: defaultImportLimits(), rule: rule,
versionState: versionState, importGraph: graph.clone(),
}
if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 {
return nil, diagnostics
}
documents := make([]*document.Node, 0, len(scope.contributions))
for _, contribution := range scope.contributions {
documents = append(documents, contribution.document)
}
composed, diagnostics := composeDocumentsWithPolicy(policy, documents...)
if composed != nil {
if diagnostic := importedVersionClaim(documents, state.scope, composed, rule); diagnostic != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
}
composition, diagnostics := composeDocumentsWithCollectionPolicy(
policy, lower, acquiredSequenceDeclarations(*scope), documents...,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
if composed != nil {
projectNestedEvidence(composed, state.scope)
projectAcquiredDestinations(composed, state.scope, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)}
composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes
composed.importGraph = state.importGraph.clone()
if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil {
return nil, []Diagnostic{*diagnostic}
}
return composed, cloneSourceInfos(state.sources), diagnostics
projectNestedEvidence(composition, *scope)
projectAcquiredDestinations(composition, *scope, sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = graph.clone()
return composition, nil
}
func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic {
@@ -547,25 +627,37 @@ func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec impo
var body *document.Node
var declarations []importDeclarationSpec
var sequenceDeclarations []sequenceDeclarationSpec
var diagnostics []Diagnostic
var processing *sourceVersionProcessingResult
directiveContext := s.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault {
versionDestination := destination
if s.itemScope {
versionDestination = appendDestination(s.itemPrefix.Index(0), destination)
}
processed, failures := s.versionState.process(root, sourceVersionContribution, versionDestination, sourceID, policy.placement)
processed, failures := s.versionState.processWithDirectiveContext(root, sourceVersionContribution, versionDestination, sourceID, policy.placement, directiveContext)
if len(failures) != 0 {
return failures
}
processing = &processed
body, declarations = processed.payload, processed.imports
} else {
body, declarations, diagnostics = splitImportDirectives(root)
if len(diagnostics) != 0 {
return diagnostics
versionDestination := destination
if s.itemScope {
versionDestination = appendDestination(s.itemPrefix.Index(0), destination)
}
processed := processSourceVersionWithDirectiveContext(root, s.rule, sourceVersionContribution, versionDestination, sourceVersionModeDefault, directiveContext)
if len(processed.diagnostics) != 0 {
return processed.diagnostics
}
processing = &processed
body, declarations = processed.payload, processed.imports
}
sequenceDeclarations = cloneSequenceDeclarations(processing.sequences)
placementDestination := destination
mode := placementModeMapping
if policy.placement == placementModeSequenceItem {
@@ -582,7 +674,7 @@ func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec impo
s.active[identity] = len(s.stack)
s.visited[identity] = route
s.stack = append(s.stack, route)
diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, processing)
diagnostics := s.visitItemSource(canonical, nodeKey, destination, relativeDestination, depth, info, body, declarations, sequenceDeclarations, processing)
delete(s.active, identity)
s.stack = s.stack[:len(s.stack)-1]
return diagnostics
@@ -601,6 +693,7 @@ func (s *importTraversal) visitPath(candidate selectedImportCandidate, spec impo
source: cloneSourceInfo(info), payload: body, document: placed,
identity: nodeKey, role: role, scopeOwner: s.scope.owner,
destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
s.active[identity] = len(s.stack)
s.visited[identity] = route
@@ -631,7 +724,7 @@ func (s *importTraversal) sourceDestination(destination Path) Path {
return s.itemFallback.clone()
}
func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic {
func (s *importTraversal) visitItemSource(canonical string, identity importNodeKey, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclarationSpec, sequenceDeclarations []sequenceDeclarationSpec, processing *sourceVersionProcessingResult) []Diagnostic {
parentScope := s.scope
parentItemScope, parentPrefix, parentFallback := s.itemScope, s.itemPrefix, s.itemFallback
s.scope = acquisitionScope{owner: identity}
@@ -645,7 +738,7 @@ func (s *importTraversal) visitItemSource(canonical string, identity importNodeK
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: body, document: body,
identity: identity, role: importRoleItem, scopeOwner: identity,
processing: processing,
processing: processing, sequenceDeclarations: cloneSequenceDeclarations(sequenceDeclarations),
})
diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{})
itemScope := s.scope
@@ -686,7 +779,13 @@ func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnos
for _, nested := range itemScope.contributions {
documents = append(documents, nested.document)
}
completed, diagnostics := composeDocuments(documents...)
// An item scope is below a sequence index. Application selectors match
// mapping-only effective paths, so none can apply inside this scope.
// Its YAML declarations remain relative to the owning record.
completed, diagnostics := composeDocumentsWithCollectionPolicy(
sequencePolicy{}, itemScope.collectionPolicies,
acquiredSequenceDeclarations(*itemScope), documents...,
)
if len(diagnostics) != 0 {
return projectItemDiagnosticPaths(diagnostics, *scope, contribution.destination, index)
}
@@ -713,6 +812,18 @@ func (s *importTraversal) composePendingItems(scope *acquisitionScope) []Diagnos
return nil
}
func acquiredSequenceDeclarations(scope acquisitionScope) []sequenceDeclarationSpec {
var declarations []sequenceDeclarationSpec
for _, contribution := range scope.contributions {
projected := cloneSequenceDeclarations(contribution.sequenceDeclarations)
for index := range projected {
projected[index].destination = appendDestination(contribution.relativeDestination, projected[index].destination)
}
declarations = append(declarations, projected...)
}
return declarations
}
func importedVersionClaim(documents []*document.Node, scope acquisitionScope, composed *composedDocument, rule *VersionRule) *Diagnostic {
if composed == nil || rule == nil {
return nil
@@ -817,8 +928,13 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
projectedApplications = append(projectedApplications, applications[sourceIndex])
continue
}
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
prefix := contribution.destination.Index(start)
itemIndex := 0
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
prefix := contribution.destination.Index(itemIndex)
if sourceIndex < len(applications) {
outer := applications[sourceIndex]
structural := document.Clone(outer.value)
@@ -840,6 +956,71 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix)
}
composed.applications = projectedApplications
projectNestedCollectionPolicies(composed, scope, ranges)
}
func projectNestedCollectionPolicies(composed *composedDocument, scope acquisitionScope, ranges map[sequenceRangeKey]sequenceRange) {
if composed == nil {
return
}
type itemPolicyProjection struct {
destination Path
policy collectionPolicy
base int
}
projected := make(map[string]*itemPolicyProjection)
var prefixLengths map[string]int
for sourceIndex, contribution := range scope.contributions {
if prefixLengths != nil && sourceIndex > 0 {
addSequencePrefixLengths(scope.contributions[sourceIndex-1].document, Path{}, prefixLengths)
}
if contribution.nested == nil || len(contribution.nested.collectionPolicies.collections) == 0 {
continue
}
key := contribution.destination.String()
assigned := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: key}]
itemIndex, found := assigned.outputForInput(0)
if !found {
// Normal composition supplies the assignment directly. For a
// handoff without ranges, scan each preceding tree only once.
if prefixLengths == nil {
prefixLengths = make(map[string]int)
for _, previous := range scope.contributions[:sourceIndex] {
addSequencePrefixLengths(previous.document, Path{}, prefixLengths)
}
}
itemIndex = prefixLengths[key]
}
working, initialized := projected[key]
if !initialized {
// Read one owned copy per destination, then add each incoming
// record without recopying earlier records and their scopes.
policy, _ := composed.collectionPolicies.at(contribution.destination)
working = &itemPolicyProjection{destination: contribution.destination, policy: policy, base: len(policy.recordOrder)}
projected[key] = working
}
working.policy.setRecordPolicyAt(working.base+itemIndex, collectionRecordPolicy{
owner: newCollectionRecordPolicy().owner,
scope: contribution.nested.collectionPolicies,
})
}
for _, working := range projected {
composed.collectionPolicies.set(working.destination, working.policy)
}
}
func addSequencePrefixLengths(node *document.Node, path Path, lengths map[string]int) {
if node == nil {
return
}
if node.Kind == document.Sequence {
lengths[path.String()] += len(node.Sequence)
return
}
// Contribution destinations select mapping paths, not sequence members.
for _, entry := range node.Mapping {
addSequencePrefixLengths(entry.Value, path.Key(entry.Key), lengths)
}
}
func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) {
@@ -875,7 +1056,12 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
}
contributionPrefix := prefix
contributionInsideItem := insideItem
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
itemIndex := 0
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}]; found {
if projected, ok := assigned.outputForInput(0); ok {
itemIndex = projected
}
}
if contribution.source.placementMode == placementModeSequenceItem {
container := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = container.clone()
@@ -885,7 +1071,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
sources[position].Destination = container
sources[position].destinationUnresolved = false
}
contributionPrefix = appendDestination(container, Path{}.Index(start))
contributionPrefix = appendDestination(container, Path{}.Index(itemIndex))
contributionInsideItem = true
projectSourceVersionEvidence(contribution.processing, contributionPrefix, nil, sourceIndex)
} else if contributionInsideItem {
@@ -928,7 +1114,9 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des
}
item, ok := schemaIndex(path.segments[position])
if ok && path.segments[position].kind != pathSegmentKey {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: assigned.start + item}
if outputIndex, projected := assigned.outputForInput(item); projected {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: outputIndex}
}
}
}
paths[index] = path

View File

@@ -115,7 +115,7 @@ tasks: [{id: z-parent}]
input, diagnostics := assembleBaseInput[config](File(mainPath), Options[config]{
Imports: true,
sequencePolicy: sequencePolicy{identityFields: map[string]string{"/tasks": "id"}},
sequencePolicy: mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("tasks")}, identityFields: []string{"id"}}}),
})
if len(diagnostics) != 0 {
t.Fatalf("assemble imports: %#v", diagnostics)
@@ -398,7 +398,7 @@ label: helper
childPath := writeImportFile(t, directory, "records/children/child.yml", "value: nested\n")
loaded, err := LoadLayered(
Input{Source: NoSource()}, testSecretBinding(),
Input{Source: NoSource(), Imports: true}, testSecretBinding(),
LayeredOptions[config]{Underlays: []Underlay{{Inputs: []Input{{
Source: File(itemPath), Imports: true, Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem,
}}}}},
@@ -525,6 +525,141 @@ func TestLoadMainItemPlacementKeepsItsHelpersInTheItem(t *testing.T) {
}
}
func TestProjectAcquiredDestinationsUsesMappedSequenceItemIndex(t *testing.T) {
child := &composedDocument{}
childScope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "nested-source"}, destination: Keys("settings"),
}}}
child.acquisition = []acquisitionScope{childScope}
composed := &composedDocument{sequenceRanges: []sequenceRange{{
sourceIndex: 0, path: Keys("records"), start: 0, end: 1,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}}}
scope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "outer-source", placementMode: placementModeSequenceItem},
destination: Keys("records"), nested: child,
}}}
sources := []SourceInfo{{ID: "outer-source"}, {ID: "nested-source"}}
projectAcquiredDestinations(composed, scope, sources)
if got, want := sources[0].Destination.String(), "/records"; got != want {
t.Errorf("outer source destination = %q, want %q", got, want)
}
if got, want := sources[1].Destination.String(), "/records/2/settings"; got != want {
t.Errorf("nested source destination = %q, want %q", got, want)
}
}
func TestProjectNestedEvidenceUsesMappedSequenceItemIndex(t *testing.T) {
outer := historyApplication{
value: parseCompositionDocument(t, "outer-source", "records: [{id: outer}]\n"),
sequences: []sequenceHistoryContribution{{
path: Keys("records"), mode: sequenceHistoryMapped,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}},
}
outer.sequencesByPath = indexSequenceHistoryContributions(outer.sequences)
nestedRoot := parseCompositionDocument(t, "nested-source", "settings: {value: nested}\n")
nestedOrigin := Origin{Layer: LayerBase, Locations: []Location{{SourceID: "nested-source"}}}
nested := &composedDocument{
root: nestedRoot, applications: []historyApplication{{value: nestedRoot}},
origins: map[string]Origin{"/settings/value": nestedOrigin},
}
scope := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "outer-source"}, destination: Keys("records"), nested: nested,
}}}
composed := &composedDocument{
applications: []historyApplication{outer},
sequenceRanges: []sequenceRange{{
sourceIndex: 0, path: Keys("records"), start: 0, end: 1,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 2}},
}},
origins: make(map[string]Origin),
}
projectNestedEvidence(composed, scope)
if got, want := composed.applications[1].path.String(), "/records/2"; got != want {
t.Fatalf("nested history application path = %q, want %q", got, want)
}
if origin := composed.origins["/records/2/settings/value"]; len(origin.Locations) != 1 || origin.Locations[0].SourceID != "nested-source" {
t.Fatalf("nested origin = %#v", origin)
}
query := Keys("records").Index(2).Key("settings").Key("value")
if operation, relevant := applicationOperationPath(composed.applications[1], query, nil); !relevant || operation.String() != query.String() {
t.Fatalf("nested historical query = %s, %v", operation.String(), relevant)
}
replayed := parseCompositionDocument(t, "lower", "records: [{id: first}, {id: middle}, {id: last}]\n")
for _, application := range composed.applications {
applyHistoryApplication(&replayed, application, new(mappingIndex))
}
if value, present := documentAtPath(replayed, query); !present || value.Text != "nested" {
t.Fatalf("replayed nested value = %#v, %v", value, present)
}
composed.applications[1].path.segments[0].key = "detached"
if got := nested.applications[0].path.String(); got != "" {
t.Fatalf("projected nested history path aliases its source: %q", got)
}
}
func TestDeferredInputCompositionUsesRetainedAcquisitionScopes(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [records]
placement: sequence-item
name: main
`)
itemPath := writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: helper.yml
id: item
`)
helperPath := writeImportFile(t, directory, "helper.yml", "value: acquired\n")
accounting := &importAccounting{}
input, diagnostics := assembleBaseInputTracked(File(mainPath), Options[struct{}]{
Imports: true, Filesystem: &FilesystemContext{Root: directory}, deferAssembly: true,
}, Path{}, accounting, nil, nil)
if len(diagnostics) != 0 {
t.Fatalf("acquire input: %#v", diagnostics)
}
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) != 1 {
t.Fatalf("acquisition handoff was composed early: %#v", input.composition)
}
if len(input.sources) != 3 || input.sources[0].ID != "source-1" || input.sources[1].ID != "source-2" || input.sources[2].ID != "source-3" {
t.Fatalf("acquired source order/identity = %#v", input.sources)
}
if _, err := os.Stat(itemPath); err != nil {
t.Fatalf("item source disappeared before composition: %v", err)
}
if _, err := os.Stat(helperPath); err != nil {
t.Fatalf("helper source disappeared before composition: %v", err)
}
composition, diagnostics := compositionForInputWithPolicy(input, sequencePolicy{})
if len(diagnostics) != 0 {
t.Fatalf("compose acquired input: %#v", diagnostics)
}
if value, present := documentAtPath(composition.root, Keys("name")); !present || value.Text != "main" {
t.Fatalf("main value = %#v, %v", value, present)
}
for path, want := range map[string]string{
"/records/0/id": "item",
"/records/0/value": "acquired",
} {
value, present := documentAtPath(composition.root, mustParsePath(t, path))
if !present || value.Text != want {
t.Errorf("composed %s = %#v, %v; want %q", path, value, present, want)
}
}
if got, want := input.sources[1].Destination.String(), "/records"; got != want {
t.Errorf("item destination = %q, want %q", got, want)
}
if got, want := input.sources[2].Destination.String(), "/records/0"; got != want {
t.Errorf("helper destination = %q, want %q", got, want)
}
}
func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) {
t.Parallel()
type config struct {
@@ -544,7 +679,7 @@ func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T)
t.Fatalf("assemble item import: %#v", diagnostics)
}
authored := parseDocumentForDecode(t, "items: [{id: authored-id, name: authored}]\n")
composed, diagnostics := composeDocumentsWithPolicy(sequencePolicy{identityFields: map[string]string{"/items": "id"}}, authored, input.document)
composed, diagnostics := composeDocumentsWithPolicy(mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}), authored, input.document)
if len(diagnostics) != 0 {
t.Fatalf("compose authored and imported lists: %#v", diagnostics)
}
@@ -874,7 +1009,7 @@ func TestItemContributionsUseDependencyReadySchedules(t *testing.T) {
type config struct {
Items []item `yaml:"items"`
}
definition := Definition[config]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}}
definition := Definition[config]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:

View File

@@ -7,10 +7,13 @@ const inheritanceKey = "extends"
// These private records keep structural evidence separate from acquired source
// records. Later inspection can project a transfer onto individual descendants.
type inheritanceTransfer struct {
from Path
to Path
reference Path
locations []Location
from Path
to Path
reference Path
locations []Location
policyTemplates []collectionPolicyScope
templatePolicies collectionPolicyScope
projectedPolicies []projectedCollectionPolicy
}
type inheritanceControl struct {
@@ -20,22 +23,110 @@ type inheritanceControl struct {
}
type inheritanceScope struct {
root *document.Node
origins map[string]Origin
transfers []inheritanceTransfer
controls []inheritanceControl
replacements []inheritanceReplacement
operations []inheritanceOperation
root *document.Node
origins map[string]Origin
collectionPolicies collectionPolicyScope
transfers []inheritanceTransfer
controls []inheritanceControl
replacements []inheritanceReplacement
operations []inheritanceOperation
}
type inheritanceOperation struct {
control *inheritanceControl
transfer *inheritanceTransfer
controlOrigin Origin
parent *document.Node
effective *document.Node
parentOrigins map[string]Origin
effectiveOrigins map[string]Origin
control *inheritanceControl
transfer *inheritanceTransfer
controlOrigin Origin
parent *document.Node
effective *document.Node
parentOrigins map[string]Origin
effectiveOrigins map[string]Origin
parentPolicies []collectionPolicyScope
templatePolicies collectionPolicyScope
effectivePolicies []collectionPolicyScope
sequences []sequenceHistoryContribution
identityFields []string
}
func cloneInheritancePolicyTemplates(templates []collectionPolicyScope) []collectionPolicyScope {
if templates == nil {
return nil
}
cloned := make([]collectionPolicyScope, len(templates))
for index, template := range templates {
cloned[index] = cloneCollectionPolicyScopeForInheritance(template)
}
return cloned
}
func cloneProjectedInheritancePolicies(policies []projectedCollectionPolicy) []projectedCollectionPolicy {
if policies == nil {
return nil
}
cloned := make([]projectedCollectionPolicy, len(policies))
for index, policy := range policies {
cloned[index] = projectedCollectionPolicy{
path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
}
}
return cloned
}
func cloneInheritanceTransfer(transfer inheritanceTransfer) inheritanceTransfer {
cloned := transfer
cloned.from = transfer.from.clone()
cloned.to = transfer.to.clone()
cloned.reference = transfer.reference.clone()
cloned.locations = append([]Location(nil), transfer.locations...)
cloned.policyTemplates = cloneInheritancePolicyTemplates(transfer.policyTemplates)
cloned.templatePolicies = cloneCollectionPolicyScopeForInheritance(transfer.templatePolicies)
cloned.projectedPolicies = cloneProjectedInheritancePolicies(transfer.projectedPolicies)
return cloned
}
func cloneInheritanceOperations(operations []inheritanceOperation) []inheritanceOperation {
if operations == nil {
return nil
}
cloned := make([]inheritanceOperation, len(operations))
for index, operation := range operations {
cloned[index] = operation
if operation.control != nil {
control := *operation.control
control.path = operation.control.path.clone()
control.locations = append([]Location(nil), operation.control.locations...)
cloned[index].control = &control
}
if operation.transfer != nil {
transfer := cloneInheritanceTransfer(*operation.transfer)
cloned[index].transfer = &transfer
}
cloned[index].parent = document.Clone(operation.parent)
cloned[index].effective = document.Clone(operation.effective)
cloned[index].parentOrigins = cloneOriginIndex(operation.parentOrigins)
cloned[index].effectiveOrigins = cloneOriginIndex(operation.effectiveOrigins)
cloned[index].parentPolicies = cloneInheritancePolicyTemplates(operation.parentPolicies)
cloned[index].templatePolicies = cloneCollectionPolicyScopeForInheritance(operation.templatePolicies)
cloned[index].effectivePolicies = cloneInheritancePolicyTemplates(operation.effectivePolicies)
cloned[index].sequences = cloneSequenceHistoryContributions(operation.sequences)
cloned[index].identityFields = cloneStrings(operation.identityFields)
}
return cloned
}
func inheritancePolicyScopeAtPath(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
return newInheritancePolicyIndex(scope).localize(prefix)
}
func projectInheritancePolicyTemplates(templates []collectionPolicyScope, prefix Path) []projectedCollectionPolicy {
var projected []projectedCollectionPolicy
for _, template := range templates {
for _, policy := range projectCollectionPolicyScope(template, prefix) {
projected = append(projected, projectedCollectionPolicy{
path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
})
}
}
return projected
}
// A replacement retains the displaced source/derivation record while the

View File

@@ -0,0 +1,309 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
func projectInheritanceCollectionPolicies(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
projected := collectionPolicyScope{}
for _, entry := range scope.collections {
policy := cloneCollectionPolicyForInheritance(entry.policy)
stripInheritanceModes(&policy)
projected.set(appendDestination(prefix, entry.destination), policy)
}
return projected
}
func projectCurrentInheritanceCollectionPolicies(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
projected := collectionPolicyScope{}
for _, entry := range scope.collections {
projected.set(appendDestination(prefix, entry.destination), cloneCollectionPolicyForInheritance(entry.policy))
}
return projected
}
func projectInheritancePolicyRoots(target *collectionPolicyScope, scope collectionPolicyScope, prefix Path) {
for _, entry := range scope.collections {
target.set(appendDestination(prefix, entry.destination), cloneCollectionPolicyForInheritance(entry.policy))
}
}
func stripInheritanceModes(policy *collectionPolicy) {
if policy == nil {
return
}
policy.explicitMode = 0
policy.explicitModePresent = false
policy.modeLocation = nil
for owner, nested := range policy.recordPolicies {
for key, entry := range nested.collections {
stripInheritanceModes(&entry.policy)
nested.collections[key] = entry
}
policy.recordPolicies[owner] = nested
}
}
func inheritanceDeclarationsAt(path Path, policy collectionPolicy) []sequenceDeclarationSpec {
if !policy.identityPresent && !policy.explicitModePresent {
return nil
}
return []sequenceDeclarationSpec{{
destination: path.clone(),
identityFields: cloneStrings(policy.identityFields), identityPresent: policy.identityPresent,
identityLocation: cloneLocation(policy.identityLocation),
mode: policy.explicitMode, modePresent: policy.explicitModePresent,
modeLocation: cloneLocation(policy.modeLocation),
}}
}
func inheritanceCollectionResolution(
path Path,
lowerScope, higherScope collectionPolicyScope,
applicationPolicy sequencePolicy,
incoming *document.Node,
) (collectionPolicyResolution, collectionPolicy, collectionPolicy, bool, []Diagnostic) {
lower, hasLower := lowerScope.at(path)
higher, hasHigher := higherScope.at(path)
applicationIdentity, registered := applicationPolicy.identityFieldsAt(path)
if !hasLower && !hasHigher && !registered {
return collectionPolicyResolution{}, collectionPolicy{}, collectionPolicy{}, false, nil
}
lowerForResolution := cloneCollectionPolicy(lower)
// The parent policy is copied to a different sibling destination. Its YAML
// identity follows the inherited collection, but its explicit mode does not.
lowerForResolution.explicitMode = 0
lowerForResolution.explicitModePresent = false
lowerForResolution.modeLocation = nil
if incoming != nil && higher.explicitModePresent && higher.explicitMode == sequenceModeReplace {
// Actual replacement discards the parent's YAML identity before
// agreement. A mode-only declaration still governs retained records.
lowerForResolution = collectionPolicy{}
}
var lowerPointer *collectionPolicy
if hasLower {
lowerPointer = &lowerForResolution
}
resolution, diagnostics := resolveCollectionPolicy(
path, inheritanceDeclarationsAt(path, higher), applicationIdentity, lowerPointer,
)
return resolution, lower, higher, true, diagnostics
}
func mergeInheritanceCollectionPolicies(
parent, child collectionPolicyScope,
parentRootPath Path,
parentValue, childValue *document.Node,
applicationPolicy sequencePolicy,
replaced collectionReplacementPaths,
) (collectionPolicyScope, []Diagnostic) {
result := cloneCollectionPolicyScopeForInheritance(parent)
paths := make(map[string]Path, len(parent.collections)+len(child.collections))
for key, entry := range parent.collections {
paths[key] = entry.destination.clone()
}
for key, entry := range child.collections {
paths[key] = entry.destination.clone()
}
var lookup *documentLookup
if len(paths) != 0 {
lookup = buildDocumentLookup(parentValue, childValue)
}
var diagnostics []Diagnostic
for _, path := range paths {
relative, inside := relativeHistoryPath(parentRootPath, path)
var retained, incoming *document.Node
if inside {
if !replaced.contains(path, false) {
retained, _ = documentAtPathWithLookup(parentValue, relative, lookup)
}
incoming, _ = documentAtPathWithLookup(childValue, relative, lookup)
}
resolution, lower, _, active, issues := inheritanceCollectionResolution(path, parent, child, applicationPolicy, incoming)
if len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
continue
}
if !active {
continue
}
var lowerPointer *collectionPolicy
if _, found := parent.at(path); found {
lower.explicitMode = 0
lower.explicitModePresent = false
lower.modeLocation = nil
lowerPointer = &lower
}
policy := resolution.policy
if incoming == nil || resolution.operation != sequenceModeReplace {
var foundDiagnostics []Diagnostic
policy, foundDiagnostics = collectionPolicyWithoutValue(lowerPointer, resolution.policy, retained, path)
if len(foundDiagnostics) != 0 {
diagnostics = append(diagnostics, foundDiagnostics...)
continue
}
}
if higher, ok := child.at(path); ok {
policy = inheritanceRecordPoliciesFromHigher(policy, higher)
}
result.set(path, policy)
}
return result, canonicalDiagnostics(diagnostics)
}
func inheritanceRecordPoliciesFromHigher(policy, higher collectionPolicy) collectionPolicy {
if len(higher.recordOrder) == 0 {
return policy
}
policy.recordOrder = nil
policy.recordPolicies = nil
for index := range higher.recordOrder {
if record, found := higher.recordPolicyAt(index); found {
policy.setRecordPolicyAt(index, currentCollectionRecordPolicy(record))
}
}
return policy
}
func inheritedCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
cloned := cloneCollectionPolicyForInheritance(collectionPolicy{
recordOrder: []*collectionRecordOwner{record.owner},
recordPolicies: map[*collectionRecordOwner]collectionPolicyScope{record.owner: record.scope},
})
if len(cloned.recordOrder) == 0 {
return newCollectionRecordPolicy()
}
owner := cloned.recordOrder[0]
scope := cloned.recordPolicies[owner]
for key, entry := range scope.collections {
stripInheritanceModes(&entry.policy)
scope.collections[key] = entry
}
return collectionRecordPolicy{owner: owner, scope: scope}
}
func currentCollectionRecordPolicy(record collectionRecordPolicy) collectionRecordPolicy {
cloned := cloneCollectionPolicyForInheritance(collectionPolicy{
recordOrder: []*collectionRecordOwner{record.owner},
recordPolicies: map[*collectionRecordOwner]collectionPolicyScope{record.owner: record.scope},
})
if len(cloned.recordOrder) == 0 {
return newCollectionRecordPolicy()
}
owner := cloned.recordOrder[0]
return collectionRecordPolicy{owner: owner, scope: cloned.recordPolicies[owner]}
}
func applyInheritanceSequencePolicy(
path Path,
combined sequenceCombinationResult,
resolution collectionPolicyResolution,
lowerScope, higherScope collectionPolicyScope,
) collectionPolicy {
policy := cloneCollectionPolicyForInheritance(resolution.policy)
policy.recordOrder = nil
policy.recordPolicies = nil
if combined.value == nil || combined.value.Kind != document.Sequence {
return policy
}
lowerPolicy, hasLower := lowerScope.at(path)
higherPolicy, hasHigher := higherScope.at(path)
lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems)
higherProjection := newSequenceItemProjectionIndex(combined.higherItems)
for outputIndex := range combined.value.Sequence {
if higherIndex, exists := higherProjection.inputForOutput(outputIndex); exists {
if record, found := higherPolicy.recordPolicyAt(higherIndex); hasHigher && found {
policy.setRecordPolicyAt(outputIndex, currentCollectionRecordPolicy(record))
} else {
policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
}
continue
}
if lowerIndex, exists := lowerProjection.inputForOutput(outputIndex); exists && hasLower {
if record, found := lowerPolicy.recordPolicyAt(lowerIndex); found {
policy.setRecordPolicyAt(outputIndex, inheritedCollectionRecordPolicy(record))
}
}
}
return policy
}
func applyInheritanceSequenceOrigins(
result *inheritanceScope,
origins, incomingOrigins map[string]Origin,
path Path,
lower, higher *document.Node,
combined sequenceCombinationResult,
operation sequenceMode,
) {
if operation == sequenceModeReplace {
if lower != nil {
displaced := displacedOrigins(lower, origins, path)
result.replacements = append(result.replacements, inheritanceReplacement{
path: path.clone(), before: lower, after: higher,
previous: displaced, effective: cloneOrigin(incomingOrigins[path.String()]),
})
deleteOriginSubtree(origins, path, lower)
}
mergeInheritanceOrigin(origins, path, incomingOrigins[path.String()])
if higher != nil && higher.Kind != document.Sequence {
copyInheritanceSubtree(origins, incomingOrigins, higher, path)
}
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
return
}
mergeInheritanceOrigin(origins, path, incomingOrigins[path.String()])
for _, replacement := range combined.replacements {
if lower == nil || lower.Kind != document.Sequence || replacement.lowerIndex < 0 || replacement.lowerIndex >= len(lower.Sequence) ||
higher == nil || higher.Kind != document.Sequence || replacement.higherIndex < 0 || replacement.higherIndex >= len(higher.Sequence) {
continue
}
lowerPath := path.Index(replacement.lowerIndex)
higherPath := path.Index(replacement.higherIndex)
outputPath := path.Index(replacement.outputIndex)
previous := displacedOrigins(lower.Sequence[replacement.lowerIndex], origins, lowerPath)
deleteOriginSubtree(origins, lowerPath, lower.Sequence[replacement.lowerIndex])
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[replacement.higherIndex], higherPath, outputPath)
result.replacements = append(result.replacements, inheritanceReplacement{
path: outputPath, before: lower.Sequence[replacement.lowerIndex], after: higher.Sequence[replacement.higherIndex],
previous: previous, effective: cloneOrigin(incomingOrigins[higherPath.String()]),
})
}
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
copyInheritanceSubtreeAt(origins, incomingOrigins, higher.Sequence[projection.inputIndex], path.Index(projection.inputIndex), path.Index(projection.outputIndex))
}
}
}
func validateRegisteredInheritanceSequences(root *document.Node, rootPath Path, applicationPolicy sequencePolicy) []Diagnostic {
var diagnostics []Diagnostic
var visit func(*document.Node, Path)
visit = func(node *document.Node, path Path) {
if node == nil {
return
}
if node.Kind == document.Sequence {
if fields, registered := applicationPolicy.identityFieldsAt(path); registered {
if _, issues := validateSequenceIdentity(node, path, fields, sequenceCombinationIdentityCodes); len(issues) != 0 {
diagnostics = append(diagnostics, issues...)
}
}
for index, item := range node.Sequence {
visit(item, path.Index(index))
}
return
}
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
visit(entry.Value, path.Key(entry.Key))
}
}
}
visit(root, rootPath)
return canonicalDiagnostics(diagnostics)
}

View File

@@ -0,0 +1,369 @@
package engine
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestInheritanceReplacementResetsOnlyReplacedYamlIdentity(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name, key, value, applicationKey string
mode sequenceMode
wantFields []string
conflict bool
}{
{name: "changed key", key: "name", value: "[{name: child}]", mode: sequenceModeReplace, wantFields: []string{"name"}},
{name: "unkeyed replacement", value: "[{name: child}]", mode: sequenceModeReplace},
{name: "non-sequence replacement", value: "42", mode: sequenceModeReplace},
{name: "agreeing application", key: "name", value: "[{name: child}]", applicationKey: "name", mode: sequenceModeReplace, wantFields: []string{"name"}},
{name: "conflicting application", key: "name", value: "[{name: child}]", applicationKey: "id", mode: sequenceModeReplace, conflict: true},
{name: "application still validates replacement", value: "[{name: child}]", applicationKey: "id", mode: sequenceModeReplace, conflict: true},
{name: "mode-only key change", key: "name", mode: sequenceModeReplace, conflict: true},
{name: "mode-only keeps identity", mode: sequenceModeReplace, wantFields: []string{"id"}},
{name: "append key change", key: "name", value: "[{name: child}]", mode: sequenceModeAppend, conflict: true},
{name: "upsert key change", key: "name", value: "[{name: child}]", mode: sequenceModeUpsert, conflict: true},
} {
t.Run(test.name, func(t *testing.T) {
input := "base: {items: [{id: parent}]}\nchild:\n extends: base\n"
if test.value != "" {
input += " items: " + test.value + "\n"
}
root := parseItemPolicyDocument(t, "replacement", input)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
higher := collectionPolicy{explicitMode: test.mode, explicitModePresent: true}
if test.key != "" {
higher.identityFields, higher.identityPresent = []string{test.key}, true
}
policies.set(Keys("child", "items"), higher)
application := sequencePolicy{}
if test.applicationKey != "" {
application = mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("child"), literalSequenceSelector("items")}, identityFields: []string{test.applicationKey},
}})
}
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, application, defaultProjectionLimits())
if test.conflict {
wantCode := CodeSequenceConflict
if test.applicationKey != "" && test.key == "" {
wantCode = CodeSequenceInvalid
}
if len(diagnostics) == 0 || diagnostics[0].Code != wantCode || diagnostics[0].Path.String() != "/child/items" && diagnostics[0].Path.String() != "/child/items/0/id" {
t.Fatalf("missing identity/constraint failure: %#v", diagnostics)
}
return
}
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, found := expanded.collectionPolicies.at(Keys("child", "items"))
if !found || !reflect.DeepEqual(policy.identityFields, test.wantFields) || policy.identityPresent != (test.wantFields != nil) {
t.Fatalf("effective YAML identity = %#v, want %v", policy, test.wantFields)
}
if test.value != "" {
actual, _ := documentAtPath(expanded.root, Keys("child", "items"))
want, _ := documentAtPath(root, Keys("child", "items"))
if !configurationValuesEqual(actual, want) {
t.Fatal("explicit replacement retained parent payload")
}
}
})
}
}
func TestLoadInheritanceReplacementUsesNewYamlKey(t *testing.T) {
t.Parallel()
loaded, err := testLoad[map[string]any](Bytes("replacement", []byte(`configkit:
sequences:
- {destination: [base, items], key: id}
- {destination: [child, items], key: name, mode: replace}
base: {items: [{id: parent}]}
child:
extends: base
items: [{name: child}]
`)))
if err != nil {
t.Fatal(err)
}
value, found := loaded.Explain(Keys("child", "items"))
if !found || value.Effective.YAML != "- name: child\n" {
t.Fatalf("replacement payload = %#v", value)
}
}
func TestInheritanceCombinesIdentifiedSequencesAtDestination(t *testing.T) {
root := parseItemPolicyDocument(t, "inheritance-values", `base:
items:
- id: shared
value: parent
child:
extends: base
items:
- id: shared
value: child
`)
modeLocation := Location{SourceID: "base-policy", Line: 4, Column: 5}
policy := collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true, modeLocation: &modeLocation,
}
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), policy)
expanded, diagnostics := expandInheritanceWithCollectionRules(
root, nil, policies, sequencePolicy{}, defaultProjectionLimits(),
)
if len(diagnostics) != 0 {
t.Fatalf("expand identified inheritance: %#v", diagnostics)
}
items, found := documentAtPath(expanded.root, Keys("child", "items"))
if !found || items.Kind != document.Sequence || len(items.Sequence) != 1 {
t.Fatalf("child items = %#v, found %t; parent mode should not force an append", items, found)
}
if value, exists := documentAtPath(items.Sequence[0], Keys("value")); !exists || value.Text != "child" {
t.Fatalf("matched record = %#v, want complete child replacement", items.Sequence[0])
}
childPolicy, found := expanded.collectionPolicies.at(Keys("child", "items"))
if !found || !childPolicy.identityPresent || childPolicy.identityFields[0] != "id" {
t.Fatalf("inherited destination identity = %#v, found %t; scope=%#v", childPolicy, found, expanded.collectionPolicies.collections)
}
if childPolicy.explicitModePresent {
t.Fatalf("parent explicit mode leaked to a different destination: %#v", childPolicy)
}
var transfer *inheritanceOperation
for index := range expanded.operations {
if expanded.operations[index].transfer != nil && expanded.operations[index].transfer.to.String() == "/child" {
transfer = &expanded.operations[index]
break
}
}
if transfer == nil || len(transfer.sequences) != 1 || transfer.sequences[0].path.String() != "/child/items" || len(transfer.sequences[0].replacements) != 1 {
t.Fatalf("inheritance sequence projection = %#v, want one mapped replacement", transfer)
}
if origin, exists := expanded.origins["/child/items/0/value"]; !exists || len(origin.Locations) == 0 || origin.Locations[0].SourceID != "inheritance-values" {
t.Fatalf("replacement origin lost the child's source context: %#v", origin)
}
}
func TestLoadAppliesYamlCollectionIdentityDuringSiblingInheritance(t *testing.T) {
input := `configkit:
sequences:
- destination: [base, items]
key: id
mode: append
base:
items:
- id: shared
value: parent
child:
extends: base
items:
- id: shared
value: child
`
loaded, err := Load[map[string]any](Bytes("inheritance-policy", []byte(input)), testSecretBinding())
if err != nil {
if loadError, ok := err.(*LoadError); ok {
t.Fatalf("load inherited collection policy: %v; diagnostics=%#v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
value, found := loaded.Inspection.Explain(Keys("child", "items").Index(0).Key("value"))
if !found || value.Effective.YAML != "child\n" {
t.Fatalf("inherited sequence value = %#v, found %t", value, found)
}
if len(value.History) == 0 {
t.Fatal("inherited sequence replacement has no inspection history")
}
parentValueVisible, childValueVisible := false, false
for _, change := range value.History {
parentValueVisible = parentValueVisible || change.Before.YAML == "parent\n" || change.After.YAML == "parent\n"
childValueVisible = childValueVisible || change.After.YAML == "child\n"
}
if !parentValueVisible || !childValueVisible {
t.Fatalf("sequence replacement history omitted its parent or child value: %#v", value.History)
}
}
func TestInheritanceUsesDestinationExplicitModeAndApplicationConstraint(t *testing.T) {
t.Run("destination append", func(t *testing.T) {
root := parseItemPolicyDocument(t, "append", `base:
items:
- id: base
child:
extends: base
items:
- id: child
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
childLocation := Location{SourceID: "child-policy", Line: 4, Column: 5}
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true, modeLocation: &childLocation,
})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand with destination append: %#v", diagnostics)
}
items, _ := documentAtPath(expanded.root, Keys("child", "items"))
if len(items.Sequence) != 2 || items.Sequence[0].Mapping[0].Value.Text != "base" || items.Sequence[1].Mapping[0].Value.Text != "child" {
t.Fatalf("appended items = %#v", items)
}
})
t.Run("new matching namespace", func(t *testing.T) {
root := parseItemPolicyDocument(t, "dynamic", `base:
enabled: true
child:
extends: base
extensions:
- id: duplicate
- id: duplicate
`)
applicationPolicy, diagnostics := compileSequenceIdentityPolicy([]sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{
{kind: sequenceSelectorLiteral, key: "child"},
{kind: sequenceSelectorLiteral, key: "extensions"},
},
identityFields: []string{"id"},
}})
if len(diagnostics) != 0 {
t.Fatalf("compile identity constraint: %#v", diagnostics)
}
_, diagnostics = expandInheritanceWithCollectionRules(root, nil, collectionPolicyScope{}, applicationPolicy, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/child/extensions" {
t.Fatalf("dynamic application constraint diagnostics = %#v", diagnostics)
}
})
}
func TestInheritanceReplacementModesAndCancellation(t *testing.T) {
for _, test := range []struct {
name string
value string
kind document.Kind
}{
{name: "empty sequence", value: "[]", kind: document.Sequence},
{name: "null", value: "null", kind: document.Null},
} {
t.Run(test.name, func(t *testing.T) {
root := parseItemPolicyDocument(t, "replacement", "base:\n items:\n - id: prior\nchild:\n extends: base\n items: "+test.value+"\n")
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeReplace, explicitModePresent: true,
})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand replacement: %#v", diagnostics)
}
items, found := documentAtPath(expanded.root, Keys("child", "items"))
if !found || items.Kind != test.kind || (items.Kind == document.Sequence && len(items.Sequence) != 0) {
t.Fatalf("replacement value = %#v, found %t", items, found)
}
})
}
for _, value := range []string{"null", "{}"} {
t.Run("append rejects "+value, func(t *testing.T) {
root := parseItemPolicyDocument(t, "append-kind", "base:\n items: [{id: prior}]\nchild:\n extends: base\n items: "+value+"\n")
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
})
_, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/child/items" {
t.Fatalf("append kind diagnostics = %#v", diagnostics)
}
})
}
t.Run("incompatible inherited identity", func(t *testing.T) {
root := parseItemPolicyDocument(t, "identity", `base:
items: [{id: one}]
child:
extends: base
items: [{id: two}]
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
policies.set(Keys("child", "items"), collectionPolicy{identityFields: []string{"slug"}, identityPresent: true})
_, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceConflict || diagnostics[0].Path.String() != "/child/items" {
t.Fatalf("identity conflict diagnostics = %#v", diagnostics)
}
})
t.Run("cancelled relationship", func(t *testing.T) {
root := parseItemPolicyDocument(t, "cancel", `base:
items: [{id: parent}]
child:
extends: null
items: [{id: local}]
`)
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand cancelled relationship: %#v", diagnostics)
}
if _, found := expanded.collectionPolicies.at(Keys("child", "items")); found {
t.Fatal("cancelled relationship transferred the parent's collection policy")
}
})
}
func TestInheritanceReplacementRetiresRecordLocalCollectionPolicy(t *testing.T) {
root := parseItemPolicyDocument(t, "record-scope", `base:
items:
- id: shared
children:
- key: old
child:
extends: base
items:
- id: shared
children:
- key: new
`)
nested := collectionPolicy{identityFields: []string{"key"}, identityPresent: true, explicitMode: sequenceModeAppend, explicitModePresent: true}
recordScope := collectionPolicyScope{}
recordScope.set(Keys("children"), nested)
outer := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
outer.setRecordPolicyAt(0, newCollectionRecordPolicy())
owner := outer.recordOrder[0]
outer.recordPolicies[owner] = recordScope
policies := collectionPolicyScope{}
policies.set(Keys("base", "items"), outer)
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand replacement record: %#v", diagnostics)
}
children, found := documentAtPath(expanded.root, Keys("child", "items").Index(0).Key("children"))
if !found || children.Kind != document.Sequence || len(children.Sequence) != 1 {
t.Fatalf("replacement children = %#v, found %t", children, found)
}
if key, exists := documentAtPath(children.Sequence[0], Keys("key")); !exists || key.Text != "new" {
t.Fatalf("replacement child = %#v, want only new record", children.Sequence[0])
}
childPolicy, exists := expanded.collectionPolicies.at(Keys("child", "items"))
if !exists {
t.Fatal("inherited outer collection policy is missing")
}
record, found := childPolicy.recordPolicyAt(0)
if !found {
t.Fatal("replacement item has no independent record owner")
}
if nestedPolicy, exists := record.scope.at(Keys("children")); exists && nestedPolicy.explicitModePresent {
t.Fatalf("replaced record retained its former nested operation: %#v", nestedPolicy)
}
}

View File

@@ -25,6 +25,12 @@ func appendInheritanceExplanation(explanation *Explanation, working **document.N
}
before := valueSnapshot(*working, query, secrets)
replaceHistoryPath(working, operation.transfer.to, operation.parent)
// A record keeps its own identity throughout inheritance. Restore it
// before taking enclosing snapshots as well as direct field snapshots.
for _, field := range operation.identityFields {
identity, _ := documentAtPath(operation.effective, Keys(field))
replaceHistoryPath(working, operation.transfer.to.Key(field), identity)
}
afterParent := valueSnapshot(*working, query, secrets)
parent, parentPresent := transferredOperationOrigin(operation, operationPathForQuery(operation.transfer.to, query))
if pathsIntersect(operation.transfer.to, query) && parentPresent {

View File

@@ -1,6 +1,7 @@
package engine
import (
"slices"
"sort"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -73,13 +74,13 @@ func applyInheritanceMetadata(metadata *Metadata, raw *document.Node, rawOrigins
})
applyOverlayMetadata(&sourceView, overlays)
metadata.history = sourceView.history
metadata.inheritance = append([]inheritanceOperation(nil), expansion.operations...)
metadata.inheritance = cloneInheritanceOperations(expansion.operations)
}
func historyWithInheritance(path Path, stored []Change, operations []inheritanceOperation) []Change {
split := len(stored)
for index, change := range stored {
if change.Origin.Layer == LayerResolution {
if change.Origin.Layer == LayerCLI || change.Origin.Layer == LayerResolution {
split = index
break
}
@@ -114,7 +115,7 @@ func projectedInheritanceChanges(query Path, operation inheritanceOperation) []C
return []Change{{Path: operation.control.path.clone(), Origin: origin,
Dependencies: []Path{operation.control.path.clone()}}}
}
if operation.transfer == nil || !pathsIntersect(operation.transfer.to, query) {
if operation.transfer == nil || !pathsIntersect(operation.transfer.to, query) || inheritanceOperationIdentity(operation, query) {
return nil
}
path := query
@@ -155,6 +156,9 @@ func projectedInheritanceChanges(query Path, operation inheritanceOperation) []C
}
func transferredOperationOrigin(operation inheritanceOperation, destination Path) (Origin, bool) {
if inheritanceOperationIdentity(operation, destination) {
return Origin{}, false
}
relative, ok := relativeHistoryPath(operation.transfer.to, destination)
if !ok {
return Origin{}, false
@@ -168,3 +172,12 @@ func transferredOperationOrigin(operation inheritanceOperation, destination Path
operation.transfer.from, operation.transfer.to, *operation.transfer)
return transferred[destination.String()], true
}
func inheritanceOperationIdentity(operation inheritanceOperation, path Path) bool {
if operation.transfer == nil || len(operation.identityFields) == 0 {
return false
}
relative, within := relativeHistoryPath(operation.transfer.to, path)
return within && len(relative.segments) != 0 && relative.segments[0].kind != pathSegmentIndex &&
slices.Contains(operation.identityFields, relative.segments[0].key)
}

View File

@@ -126,7 +126,7 @@ func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocume
applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded)
finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root)
return &Inspection{
metadata: metadata, baseline: baseline, input: expanded.root,
metadata: metadata, baseline: baseline, input: expanded.root, preCLI: expanded.root,
inputOrigins: expanded.origins, final: expanded.root,
applications: composed.applications, inheritance: expanded.operations,
}, expanded

View File

@@ -0,0 +1,172 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
type identifiedInheritanceTemplate struct {
item *document.Node
path Path
tuple string
parent int
hasParent bool
cancelled bool
directive *document.Node
directivePath Path
}
type identifiedInheritanceGraph struct {
scope *document.Node
scopePath Path
fields []string
nodes []identifiedInheritanceTemplate
edges []dependencyEdge
byIdentity map[string]int
}
func sequenceHasIdentifiedInheritanceReferences(sequence *document.Node) bool {
if sequence == nil || sequence.Kind != document.Sequence {
return false
}
for _, item := range sequence.Sequence {
if directive, found := inheritanceDirective(item); found && directive.Value.Kind != document.Null {
return true
}
}
return false
}
// analyzeIdentifiedInheritanceGraph validates relationships inside one
// identified sequence. It prepares a graph only; mapping-sibling expansion
// remains the active inheritance behavior until the reusable scope executor is
// activated.
func analyzeIdentifiedInheritanceGraph(sequence *document.Node, scopePath Path, identityFields []string) (*identifiedInheritanceGraph, []Diagnostic) {
if sequence == nil || sequence.Kind != document.Sequence {
var primary *Location
if sequence != nil {
location := publicLocation(sequence.Location)
primary = &location
}
return nil, []Diagnostic{{
Code: CodeSequenceInvalid, Severity: SeverityError, Path: scopePath.clone(),
Message: "identified inheritance scope must be a sequence", Primary: primary,
}}
}
fields, err := normalizeSequenceIdentityFields(identityFields)
if err != nil {
location := publicLocation(sequence.Location)
return nil, []Diagnostic{{
Code: CodeSequenceInvalid, Severity: SeverityError, Path: scopePath.clone(),
Message: "identified inheritance scope requires identity fields", Primary: &location,
}}
}
lookup, diagnostics := validateNormalizedSequenceIdentityFrom(
sequence, scopePath, fields, sequenceCombinationIdentityCodes, 0, sequenceStructuralRecord,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
graph := &identifiedInheritanceGraph{
scope: sequence, scopePath: scopePath.clone(), fields: cloneStrings(fields),
nodes: make([]identifiedInheritanceTemplate, len(sequence.Sequence)),
byIdentity: make(map[string]int, len(lookup)),
}
for tuple, record := range lookup {
graph.byIdentity[tuple] = record.index
graph.nodes[record.index] = identifiedInheritanceTemplate{
item: sequence.Sequence[record.index], path: scopePath.Index(record.index), tuple: tuple,
}
}
for index := range graph.nodes {
node := &graph.nodes[index]
directive, found := inheritanceDirective(node.item)
if !found {
continue
}
node.directive = directive.Value
node.directivePath = node.path.Key(inheritanceKey)
referenceLocation := publicLocation(directive.Value.Location)
switch directive.Value.Kind {
case document.Null:
node.cancelled = true
case document.Mapping:
reference, issues := identifiedInheritanceReference(directive.Value, node.directivePath, fields)
if len(issues) != 0 {
return nil, issues
}
parent, exists := graph.byIdentity[reference]
if !exists {
return nil, []Diagnostic{inheritanceDiagnostic(
CodeInheritanceMissing, node.directivePath, referenceLocation, nil,
"referenced record does not exist in this collection",
)}
}
node.parent = parent
node.hasParent = true
graph.edges = append(graph.edges, dependencyEdge{prerequisite: parent, dependent: index})
default:
return nil, []Diagnostic{inheritanceDiagnostic(
CodeInheritanceInvalid, node.directivePath, referenceLocation, nil,
"identified record inheritance reference must be a mapping or null",
)}
}
}
return graph, nil
}
func identifiedInheritanceReference(reference *document.Node, path Path, identityFields []string) (string, []Diagnostic) {
allowed := make(map[string]struct{}, len(identityFields))
for _, field := range identityFields {
allowed[field] = struct{}{}
}
seen := make(map[string]struct{}, len(reference.Mapping))
for _, entry := range reference.Mapping {
if _, ok := allowed[entry.Key]; !ok {
return "", []Diagnostic{inheritanceDiagnostic(
CodeInheritanceInvalid, path.Key(entry.Key), publicLocation(entry.Value.Location), nil,
"identified record inheritance reference has an unsupported field",
)}
}
seen[entry.Key] = struct{}{}
}
for _, field := range identityFields {
if _, ok := seen[field]; !ok {
return "", []Diagnostic{inheritanceDiagnostic(
CodeInheritanceInvalid, path.Key(field), publicLocation(reference.Location), nil,
"identified record inheritance reference is missing an identity field",
)}
}
}
index := newSequenceMappingIndex(reference, sequencePayloadRecord)
tuple, _, missing, invalid := index.identity(identityFields)
if missing != "" || invalid >= 0 || invalid == -2 {
value := reference
fieldPath := path.Key(missing)
if invalid >= 0 && invalid < len(reference.Mapping) {
value = reference.Mapping[invalid].Value
fieldPath = path.Key(reference.Mapping[invalid].Key)
}
location := publicLocation(value.Location)
return "", []Diagnostic{inheritanceDiagnostic(
CodeInheritanceInvalid, fieldPath, location, nil,
"identified record inheritance identity fields must be scalar values",
)}
}
return tuple, nil
}
func (graph *identifiedInheritanceGraph) schedule() ([]int, []Diagnostic) {
keys := make([]dependencyKey, len(graph.nodes))
for index, node := range graph.nodes {
keys[index] = dependencyKey{"record", graph.scopePath.String(), node.path.String()}
}
order, cycle := scheduleDependencies(keys, graph.edges)
if len(cycle) == 0 {
return order, nil
}
parent, child := graph.nodes[cycle[0]], graph.nodes[cycle[1]]
return nil, []Diagnostic{inheritanceDiagnostic(
CodeInheritanceCycle, child.directivePath, publicLocation(child.directive.Location),
[]Location{publicLocation(parent.item.Location)},
"identified record inheritance contains a cycle",
)}
}

View File

@@ -0,0 +1,373 @@
package engine
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func parseIdentifiedInheritanceSequence(t *testing.T, sourceID, input string) *document.Node {
t.Helper()
var indented strings.Builder
indented.WriteString("jobs:\n")
for _, line := range strings.Split(strings.TrimSuffix(input, "\n"), "\n") {
indented.WriteString(" ")
indented.WriteString(line)
indented.WriteByte('\n')
}
root := parseItemPolicyDocument(t, sourceID, indented.String())
sequence, found := documentAtPath(root, Keys("jobs"))
if !found || sequence.Kind != document.Sequence {
t.Fatalf("identified sequence missing from test document: %#v", root)
}
return sequence
}
func TestIdentifiedInheritanceGraphSchedulesForwardReferencesInPlace(t *testing.T) {
sequence := parseIdentifiedInheritanceSequence(t, "identified-graph", `- id: 2
tenant: acme
extends: {id: 1.0, tenant: acme}
- id: 1e0
tenant: acme
extends: {id: 0, tenant: acme}
- id: 0
tenant: acme
- id: 3
tenant: acme
extends: null
`)
path := Keys("jobs")
graph, diagnostics := analyzeIdentifiedInheritanceGraph(sequence, path, []string{"tenant", "id"})
if len(diagnostics) != 0 {
t.Fatalf("analyze identified graph: %#v", diagnostics)
}
if len(graph.edges) != 2 {
t.Fatalf("edges = %#v, want the two links in the inheritance chain", graph.edges)
}
order, diagnostics := graph.schedule()
if len(diagnostics) != 0 {
t.Fatalf("schedule identified graph: %#v", diagnostics)
}
if len(order) != 4 || order[0] != 2 || order[1] != 1 || order[2] != 0 {
t.Fatalf("schedule = %v, want the forward parent chain before its children", order)
}
if graph.nodes[0].path.String() != "/jobs/0" || graph.nodes[0].item != sequence.Sequence[0] ||
!graph.nodes[3].cancelled || len(graph.nodes[3].directivePath.segments) == 0 {
t.Fatalf("graph nodes changed authored positions or cancellation: %#v", graph.nodes)
}
if _, found := graph.byIdentity[graph.nodes[1].tuple]; !found {
t.Fatal("identity index does not contain the parent record")
}
}
func TestIdentifiedInheritanceScopeAdapterReusesTemplateExecution(t *testing.T) {
sequence := parseIdentifiedInheritanceSequence(t, "identified-scope", `- id: child-a
tenant: team-a
extends: {id: parent-a, tenant: team-a}
settings: {child: a}
- id: parent-a
tenant: team-a
settings: {parent: a}
- id: child-b
tenant: team-b
extends: {id: parent-b, tenant: team-b}
settings: {child: b}
- id: parent-b
tenant: team-b
settings: {parent: b}
`)
path := Keys("jobs")
graph, diagnostics := analyzeIdentifiedInheritanceGraph(sequence, path, []string{"tenant", "id"})
if len(diagnostics) != 0 {
t.Fatalf("analyze identified graph: %#v", diagnostics)
}
policies := collectionPolicyScope{}
policies.set(path.Index(1).Key("steps"), collectionPolicy{
identityFields: []string{"step-id"}, identityPresent: true,
})
result, diagnostics := applyIdentifiedInheritanceTemplates(graph, defaultProjectionLimits(), nil, policies, sequencePolicy{})
if len(diagnostics) != 0 {
t.Fatalf("apply identified templates: %#v", diagnostics)
}
if got := len(result.root.Sequence); got != 4 {
t.Fatalf("result sequence length = %d, want 4", got)
}
for index, want := range []string{"child-a", "parent-a", "child-b", "parent-b"} {
id, found := documentAtPath(result.root.Sequence[index], Path{}.Key("id"))
if !found || id.Text != want {
t.Fatalf("result identity at index %d = %#v, found %t; want %q", index, id, found, want)
}
}
for index, want := range []string{"a", "b"} {
value, found := documentAtPath(result.root.Sequence[index*2], Path{}.Key("settings").Key("parent"))
if !found || value.Text != want {
t.Fatalf("inherited value at record %d = %#v, found %t; want %q", index*2, value, found, want)
}
}
if len(result.transfers) != 2 || result.transfers[0].from.String() != "/jobs/1" || result.transfers[0].to.String() != "/jobs/0" ||
result.transfers[1].from.String() != "/jobs/3" || result.transfers[1].to.String() != "/jobs/2" {
t.Fatalf("record transfers = %#v, want independent transfers at authored positions", result.transfers)
}
if len(result.controls) != 2 || result.controls[0].path.String() != "/jobs/0/extends" || result.controls[1].path.String() != "/jobs/2/extends" {
t.Fatalf("relationship controls = %#v, want paths from child positions", result.controls)
}
if len(result.operations) != 4 || result.operations[1].transfer == nil || result.operations[1].transfer.to.String() != "/jobs/0" ||
result.operations[3].transfer == nil || result.operations[3].transfer.to.String() != "/jobs/2" {
t.Fatalf("relationship history operations = %#v, want transfer evidence at child positions", result.operations)
}
if _, found := result.origins[path.Index(0).Key("id").String()]; !found {
t.Fatal("resolved sequence output did not retain the child identity origin path")
}
projectedPolicy, found := result.collectionPolicies.at(path.Index(0).Key("steps"))
if !found || !projectedPolicy.identityPresent || len(projectedPolicy.identityFields) != 1 || projectedPolicy.identityFields[0] != "step-id" {
t.Fatalf("inherited nested collection policy = %#v, found %t", projectedPolicy, found)
}
adapter := identifiedSequenceInheritanceScope{graph: graph, policies: policies}
fields := adapter.templates()[0].identityFields
if len(fields) != 2 || fields[0] != "id" || fields[1] != "tenant" {
t.Fatalf("payload identity fields = %v, want [id tenant]", fields)
}
templates := adapter.templates()
parentPolicy, found := templates[1].collectionPolicies.at(Keys("steps"))
if !found || len(parentPolicy.identityFields) != 1 || parentPolicy.identityFields[0] != "step-id" {
t.Fatalf("parent nested policy = %#v, found %t", parentPolicy, found)
}
if _, found := templates[0].collectionPolicies.at(Keys("steps")); found {
t.Fatal("nested policy from one record leaked into a sibling template")
}
parentPolicy.identityFields[0] = "changed"
templates[1].collectionPolicies.set(Keys("steps"), parentPolicy)
reused, found := adapter.templates()[1].collectionPolicies.at(Keys("steps"))
if !found || len(reused.identityFields) != 1 || reused.identityFields[0] != "step-id" {
t.Fatal("reusing the adapter observed a mutation to a prior template snapshot")
}
if sequence.Sequence[0].Mapping[0].Value.Text != "child-a" {
t.Fatal("executing the sequence adapter mutated its source scope")
}
}
func TestIdentifiedInheritanceResolvesNestedCollectionsAtEachDestination(t *testing.T) {
root := parseItemPolicyDocument(t, "identified-nested", `jobs:
- id: child
extends: {id: parent}
- id: parent
extends: {id: base}
steps:
- id: child-step
extends: {id: parent-step}
value: child-value
- id: parent-step
value: parent-value
- id: base
steps:
- id: base-step
value: base-value
`)
identity := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
policies := collectionPolicyScope{}
policies.set(Keys("jobs"), identity)
for index := 0; index < 3; index++ {
policies.set(Keys("jobs").Index(index).Key("steps"), identity)
}
expanded, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("expand nested identified inheritance: %#v", diagnostics)
}
steps, found := documentAtPath(expanded.root, Keys("jobs").Index(0).Key("steps"))
if !found || steps.Kind != document.Sequence || len(steps.Sequence) != 3 {
t.Fatalf("child nested steps = %#v, found %t", steps, found)
}
for index, want := range []string{"base-step", "child-step", "parent-step"} {
id, exists := documentAtPath(steps.Sequence[index], Keys("id"))
if !exists || id.Text != want {
t.Fatalf("child step %d identity = %#v, found %t; want %q", index, id, exists, want)
}
}
value, found := documentAtPath(steps.Sequence[1], Keys("value"))
if !found || value.Text != "child-value" {
t.Fatalf("nested record inheritance = %#v, found %t", value, found)
}
if containsInheritanceDirective(expanded.root) {
t.Fatal("a nested record relationship remained after destination-local expansion")
}
childIdentity, found := expanded.origins["/jobs/0/id"]
if !found || len(childIdentity.Locations) == 0 || childIdentity.Locations[0].SourceID != "identified-nested" {
t.Fatalf("child identity origin = %#v, found %t", childIdentity, found)
}
if _, found := expanded.origins["/jobs/0/steps/1/value"]; !found {
t.Fatal("nested inherited value did not retain its destination origin")
}
for _, replacement := range expanded.replacements {
if replacement.path.String() == "/jobs/0/id" {
t.Fatal("parent identity was treated as payload replaced by the child's identity")
}
}
}
func TestIdentifiedInheritanceRespectsOutputNodeBudget(t *testing.T) {
root := parseItemPolicyDocument(t, "identified-budget", `jobs:
- id: child
extends: {id: parent}
- id: parent
value: inherited
`)
policies := collectionPolicyScope{}
policies.set(Keys("jobs"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
baseline, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, defaultProjectionLimits())
if len(diagnostics) != 0 {
t.Fatalf("unbounded identified inheritance: %#v", diagnostics)
}
outputNodes, within := inheritanceLowerBoundCount(baseline.root, -1)
if !within || outputNodes < 2 {
t.Fatalf("identified output node count = %d, within %t", outputNodes, within)
}
limits := defaultProjectionLimits()
limits.nodes = outputNodes
if result, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, limits); result == nil || len(diagnostics) != 0 {
t.Fatalf("at output-node boundary: result = %#v, diagnostics = %#v", result, diagnostics)
}
limits.nodes = outputNodes - 1
if result, diagnostics := expandInheritanceWithCollectionRules(root, nil, policies, sequencePolicy{}, limits); result != nil ||
len(diagnostics) != 1 || diagnostics[0].Code != CodeSchemaProjectionLimit {
t.Fatalf("below output-node boundary: result = %#v, diagnostics = %#v", result, diagnostics)
}
}
func TestIdentifiedInheritanceRequiresChildIdentityBeforeParentLookup(t *testing.T) {
sequence := parseIdentifiedInheritanceSequence(t, "missing-child-field", `- id: child
extends: {id: parent, tenant: acme}
- id: parent
tenant: acme
`)
_, diagnostics := analyzeIdentifiedInheritanceGraph(sequence, Keys("jobs"), []string{"id", "tenant"})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid || diagnostics[0].Path.String() != "/jobs/0/tenant" {
t.Fatalf("missing child identity diagnostics = %#v", diagnostics)
}
}
func TestIdentifiedInheritanceRejectsMalformedAndMissingReferences(t *testing.T) {
tests := []struct {
name string
value string
wantCode Code
}{
{name: "string shorthand", value: "extends: parent\n", wantCode: CodeInheritanceInvalid},
{name: "sequence shorthand", value: "extends: [{id: parent, tenant: acme}]\n", wantCode: CodeInheritanceInvalid},
{name: "missing component", value: "extends: {id: parent}\n", wantCode: CodeInheritanceInvalid},
{name: "extra component", value: "extends: {id: parent, tenant: acme, extra: value}\n", wantCode: CodeInheritanceInvalid},
{name: "mapping component", value: "extends: {id: {nested: value}, tenant: acme}\n", wantCode: CodeInheritanceInvalid},
{name: "missing parent", value: "extends: {id: missing, tenant: acme}\n", wantCode: CodeInheritanceMissing},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sequence := parseIdentifiedInheritanceSequence(t, "bad-reference", "- id: child\n tenant: acme\n "+test.value)
_, diagnostics := analyzeIdentifiedInheritanceGraph(sequence, Keys("jobs"), []string{"id", "tenant"})
if len(diagnostics) != 1 || diagnostics[0].Code != test.wantCode || diagnostics[0].Path.String() != "/jobs/0/extends" && diagnostics[0].Code != CodeInheritanceInvalid {
t.Fatalf("diagnostics = %#v", diagnostics)
}
if diagnostics[0].Code == CodeInheritanceMissing && diagnostics[0].Path.String() != "/jobs/0/extends" {
t.Fatalf("missing-parent path = %s", diagnostics[0].Path)
}
})
}
}
func TestIdentifiedInheritanceGraphFindsSelfAndLongerCycles(t *testing.T) {
for _, input := range []string{
`- id: self
tenant: acme
extends: {id: self, tenant: acme}
`,
`- id: alpha
tenant: acme
extends: {id: beta, tenant: acme}
- id: beta
tenant: acme
extends: {id: gamma, tenant: acme}
- id: gamma
tenant: acme
extends: {id: alpha, tenant: acme}
`,
} {
graph, diagnostics := analyzeIdentifiedInheritanceGraph(parseIdentifiedInheritanceSequence(t, "cycle", input), Keys("jobs"), []string{"id", "tenant"})
if len(diagnostics) != 0 {
t.Fatalf("analyze cycle graph: %#v", diagnostics)
}
_, diagnostics = graph.schedule()
if len(diagnostics) != 1 || diagnostics[0].Code != CodeInheritanceCycle || diagnostics[0].Path.String() == "" {
t.Fatalf("cycle diagnostics = %#v", diagnostics)
}
if diagnostics[0].Primary == nil || diagnostics[0].Primary.SourceID != "cycle" {
t.Fatalf("cycle primary location = %#v", diagnostics[0].Primary)
}
}
}
func TestIdentifiedInheritanceScopesCannotReferenceOtherCollections(t *testing.T) {
other := parseIdentifiedInheritanceSequence(t, "other-collection", `- id: parent
tenant: acme
`)
_, diagnostics := analyzeIdentifiedInheritanceGraph(other, Keys("other"), []string{"id", "tenant"})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
first := parseIdentifiedInheritanceSequence(t, "first-collection", `- id: child
tenant: acme
extends: {id: parent, tenant: acme}
`)
_, diagnostics = analyzeIdentifiedInheritanceGraph(first, Keys("first"), []string{"id", "tenant"})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeInheritanceMissing {
t.Fatalf("cross-collection reference diagnostics = %#v", diagnostics)
}
}
func TestIdentifiedInheritanceAcceptsStrictlyComposedParentReference(t *testing.T) {
left := parseCompositionDocument(t, "reference-left", "extends: {id: parent}\n")
right := parseCompositionDocument(t, "reference-right", "extends: {tenant: acme}\n")
reference, diagnostics := composeDocuments(left, right)
if len(diagnostics) != 0 {
t.Fatalf("compose disjoint reference helpers: %#v", diagnostics)
}
child, diagnostics := composeDocuments(parseCompositionDocument(t, "child-fields", "id: child\ntenant: acme\n"), reference.root)
if len(diagnostics) != 0 {
t.Fatalf("compose child fields and reference: %#v", diagnostics)
}
parent := parseCompositionDocument(t, "parent", "id: parent\ntenant: acme\n")
sequence := &document.Node{Kind: document.Sequence, Sequence: []*document.Node{child.root, parent}}
graph, diagnostics := analyzeIdentifiedInheritanceGraph(sequence, Keys("jobs"), []string{"id", "tenant"})
if len(diagnostics) != 0 {
t.Fatalf("analyze composed reference: %#v", diagnostics)
}
if len(graph.edges) != 1 || graph.edges[0] != (dependencyEdge{prerequisite: 1, dependent: 0}) {
t.Fatalf("composed reference edge = %#v", graph.edges)
}
_, diagnostics = composeDocuments(
parseCompositionDocument(t, "overlap-a", "extends: {id: first}\n"),
parseCompositionDocument(t, "overlap-b", "extends: {id: second}\n"),
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeConflict {
t.Fatalf("overlapping reference fields should conflict during strict composition: %#v", diagnostics)
}
}
func TestIdentifiedInheritanceValidatesOnlySurvivingRelationships(t *testing.T) {
lower := parseIdentifiedInheritanceSequence(t, "superseded", `- id: shared
extends: {id: absent, tenant: acme}
`)
higher := parseIdentifiedInheritanceSequence(t, "cancelled", `- id: shared
extends: null
`)
resolution := collectionPolicyResolution{
operation: sequenceModeUpsert, effectiveIdentityFields: []string{"id"},
}
combined, diagnostics := combineSequence(lower, higher, resolution, sequenceCombinationContext{destination: Keys("jobs")})
if len(diagnostics) != 0 {
t.Fatalf("combine surviving reference: %#v", diagnostics)
}
graph, diagnostics := analyzeIdentifiedInheritanceGraph(combined.value, Keys("jobs"), []string{"id"})
if len(diagnostics) != 0 || len(graph.edges) != 0 || !graph.nodes[0].cancelled {
t.Fatalf("superseded missing reference was validated: graph=%#v diagnostics=%#v", graph, diagnostics)
}
}

View File

@@ -0,0 +1,87 @@
package engine
import (
"fmt"
"strings"
"testing"
)
func TestRecordInheritanceHistoryPreservesChildIdentity(t *testing.T) {
t.Parallel()
for _, compound := range []bool{false, true} {
t.Run(fmt.Sprintf("compound=%t", compound), func(t *testing.T) {
key, reference := "id", "{id: base}"
identities := map[string]string{"id": "child"}
if compound {
key, reference = "[id, tenant]", "{id: base, tenant: parent-team}"
identities["tenant"] = "child-team"
}
body := "configkit:\n sequences: [{destination: [items], key: " + key + "}]\nitems:\n - id: base\n tenant: parent-team\n value: inherited\n - id: child\n tenant: child-team\n extends: " + reference + "\n"
loaded, err := testLoad(Bytes("records", []byte(body)), Options[map[string]any]{})
if err != nil {
t.Fatal(err)
}
child := Keys("items").Index(1)
for field, want := range identities {
path := child.Key(field)
info, found := loaded.Metadata().Lookup(path)
if !found || !info.Explicit || len(info.Origin.Inheritance) != 0 {
t.Fatalf("child identity provenance = %#v, found %t", info, found)
}
for _, change := range loaded.Metadata().History(path) {
if change.Origin.Operation == ChangeInheritance {
t.Fatalf("child identity received inheritance history: %#v", change)
}
}
explanation, found := loaded.Explain(path)
if !found {
t.Fatalf("missing identity explanation at %s", path)
}
for _, change := range explanation.History {
for _, snapshot := range []ValueSnapshot{change.Before, change.After} {
if snapshot.Present && snapshot.YAML != want+"\n" {
t.Fatalf("child identity snapshot = %q, want %q", snapshot.YAML, want)
}
}
}
}
for _, query := range []Path{child, Keys("items"), {}} {
explanation, found := loaded.Explain(query)
if !found {
t.Fatalf("missing ancestor explanation at %s", query)
}
relative, _ := relativeHistoryPath(query, child)
for _, change := range explanation.History {
for _, snapshot := range []ValueSnapshot{change.Before, change.After} {
if !snapshot.Present {
continue
}
wrapped := "snapshot:\n " + strings.ReplaceAll(snapshot.YAML, "\n", "\n ") + "\n"
root := parseItemPolicyDocument(t, "history", wrapped)
for field, want := range identities {
value, present := documentAtPath(root, appendDestination(Keys("snapshot"), relative.Key(field)))
if present && value.Text != want {
t.Fatalf("ancestor %s snapshot has child %s = %q", query, field, value.Text)
}
}
}
}
}
payload, found := loaded.Explain(child.Key("value"))
if !found || payload.Effective.YAML != "inherited\n" {
t.Fatalf("inherited payload = %#v, found %t", payload, found)
}
inherited := false
for _, change := range payload.History {
inherited = inherited || change.Origin.Operation == ChangeInheritance
}
if !inherited {
t.Fatal("ordinary inherited payload lost its history")
}
parent, found := loaded.Explain(Keys("items").Index(0).Key("id"))
if !found || parent.Effective.YAML != "base\n" || len(parent.History) == 0 {
t.Fatalf("parent identity history = %#v, found %t", parent, found)
}
})
}
}

View File

@@ -27,11 +27,12 @@ func containsInheritanceDirective(root *document.Node) bool {
return visit(root)
}
func inheritanceVersionDiagnostic(rule *VersionRule, expanded *inheritanceScope) *Diagnostic {
func projectInheritanceVersion(rule *VersionRule, expanded *inheritanceScope) (*VersionRule, *Diagnostic) {
if rule == nil || expanded == nil {
return nil
return rule, nil
}
for _, operation := range expanded.operations {
rule = projectVersionRule(rule, operation.sequences)
var location Location
var affected bool
if operation.control != nil {
@@ -50,18 +51,18 @@ func inheritanceVersionDiagnostic(rule *VersionRule, expanded *inheritanceScope)
}
}
if affected {
return &Diagnostic{
return rule, &Diagnostic{
Code: CodeInheritanceVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "inheritance cannot change the schema-version path", Primary: &location,
}
}
}
return nil
return rule, nil
}
func inheritanceOperations(expanded *inheritanceScope) []inheritanceOperation {
if expanded == nil {
return nil
}
return expanded.operations
return cloneInheritanceOperations(expanded.operations)
}

View File

@@ -70,8 +70,14 @@ func mergeInheritanceOrigin(origins map[string]Origin, path Path, incoming Origi
}
func copyInheritanceSubtree(target, incoming map[string]Origin, node *document.Node, root Path) {
walkDocument(node, root, func(path Path, _ *document.Node) {
target[path.String()] = cloneOrigin(incoming[path.String()])
copyInheritanceSubtreeAt(target, incoming, node, root, root)
}
func copyInheritanceSubtreeAt(target, incoming map[string]Origin, node *document.Node, from, to Path) {
walkDocument(node, Path{}, func(relative Path, _ *document.Node) {
source := appendDestination(from, relative)
destination := appendDestination(to, relative)
target[destination.String()] = cloneOrigin(incoming[source.String()])
})
}

View File

@@ -0,0 +1,123 @@
package engine
// inheritancePolicyIndex borrows immutable policy entries. Semantic path
// segments keep mapping keys distinct from sequence indexes. Record-owned
// scopes have their own indexes, just as they have their own policy tables.
type inheritancePolicyIndex struct {
entry *collectionPolicyEntry
children map[pathSegment]*inheritancePolicyIndex
records map[int]*inheritancePolicyIndex
}
func newInheritancePolicyIndex(scope collectionPolicyScope) *inheritancePolicyIndex {
index := &inheritancePolicyIndex{}
for _, entry := range scope.collections {
index.set(entry)
}
return index
}
func (index *inheritancePolicyIndex) set(entry collectionPolicyEntry) {
node := index
for _, segment := range entry.destination.segments {
if node.children == nil {
node.children = make(map[pathSegment]*inheritancePolicyIndex)
}
if node.children[segment] == nil {
node.children[segment] = &inheritancePolicyIndex{}
}
node = node.children[segment]
}
node.entry = &entry
node.records = nil
for position, owner := range entry.policy.recordOrder {
scope := entry.policy.recordPolicies[owner]
if len(scope.collections) == 0 {
continue
}
if node.records == nil {
node.records = make(map[int]*inheritancePolicyIndex)
}
node.records[position] = newInheritancePolicyIndex(scope)
}
}
// localize copies only policies in the requested scope. Earlier record-owned
// policies are considered before direct entries added by destination expansion.
func (index *inheritancePolicyIndex) localize(prefix Path) collectionPolicyScope {
localized := collectionPolicyScope{}
node := index
for offset, segment := range prefix.segments {
if node == nil {
return localized
}
if segment.kind == pathSegmentIndex {
if record := node.records[segment.index]; record != nil {
remainder := Path{segments: prefix.segments[offset+1:]}
for _, entry := range record.localize(remainder).collections {
localized.set(entry.destination, entry.policy)
}
}
}
node = node.children[segment]
}
var collect func(*inheritancePolicyIndex)
collect = func(node *inheritancePolicyIndex) {
if node == nil {
return
}
if node.entry != nil {
relative, _ := relativeHistoryPath(prefix, node.entry.destination)
localized.set(relative, cloneCollectionPolicyForInheritance(node.entry.policy))
}
for _, child := range node.children {
collect(child)
}
}
collect(node)
return localized
}
func (index *inheritancePolicyIndex) absoluteScope(prefix Path) collectionPolicyScope {
scope := collectionPolicyScope{}
for _, entry := range index.localize(prefix).collections {
scope.set(appendDestination(prefix, entry.destination), entry.policy)
}
return scope
}
// replace updates the walker's owned table and its borrowed index together.
// Retirement visits only direct entries below the changed destination.
func (index *inheritancePolicyIndex) replace(scope *collectionPolicyScope, prefix Path, replacement collectionPolicyScope) {
node := index
var parent *inheritancePolicyIndex
var last pathSegment
for _, segment := range prefix.segments {
if node == nil {
break
}
parent, last, node = node, segment, node.children[segment]
}
var remove func(*inheritancePolicyIndex)
remove = func(node *inheritancePolicyIndex) {
if node == nil {
return
}
if node.entry != nil {
delete(scope.collections, node.entry.destination.String())
}
for _, child := range node.children {
remove(child)
}
}
remove(node)
if parent == nil {
*index = inheritancePolicyIndex{}
} else {
delete(parent.children, last)
}
for _, entry := range replacement.collections {
scope.set(entry.destination, entry.policy)
index.set(scope.collections[entry.destination.String()])
}
}

View File

@@ -0,0 +1,111 @@
package engine
import (
"fmt"
"strings"
"testing"
)
func TestInheritancePolicyIndexKeepsSemanticScopesAndOwnedTemplates(t *testing.T) {
policy := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
nested := collectionPolicyScope{}
nested.set(Keys("group", "base", "items"), policy)
outer := policy
record := newCollectionRecordPolicy()
record.scope = nested
outer.setRecordPolicyAt(0, record)
scope := collectionPolicyScope{}
scope.set(Keys("records"), outer)
scope.set(Keys("numeric", "0", "items"), policy)
index := newInheritancePolicyIndex(scope)
first := index.localize(Keys("records").Index(0).Key("group"))
second := index.localize(Keys("records").Index(0).Key("group"))
if len(first.collections) != 1 || len(second.collections) != 1 {
t.Fatal("record-owned scope was not localized")
}
entry := first.collections["/base/items"]
entry.policy.identityFields[0] = "changed"
if second.collections["/base/items"].policy.identityFields[0] != "id" || nested.collections["/group/base/items"].policy.identityFields[0] != "id" {
t.Fatal("localized policy shares mutable template state")
}
if len(index.localize(Keys("records", "0")).collections) != 0 || len(index.localize(Keys("numeric").Index(0)).collections) != 0 {
t.Fatal("mapping keys and sequence indexes were conflated")
}
if len(index.localize(Keys("numeric", "0")).collections) != 1 {
t.Fatal("numeric mapping key did not retain its policy")
}
left := index.localize(Keys("records"))
right := index.localize(Keys("records"))
leftOwner := left.collections[""].policy.recordOrder[0]
rightOwner := right.collections[""].policy.recordOrder[0]
if leftOwner == rightOwner || leftOwner == record.owner || rightOwner == record.owner {
t.Fatal("independent templates share a record owner")
}
}
func TestInheritancePolicyIndexReplacesOnlyTheChangedScope(t *testing.T) {
scope := collectionPolicyScope{}
policy := collectionPolicy{explicitMode: sequenceModeAppend, explicitModePresent: true}
for index := range 128 {
scope.set(Keys(fmt.Sprintf("group-%03d", index), "pending"), policy)
}
index := newInheritancePolicyIndex(scope)
changed := Keys("group-064")
if len(index.absoluteScope(changed).collections) != 1 {
t.Fatal("scope includes unrelated groups")
}
replacement := collectionPolicyScope{}
replacement.set(changed.Key("introduced"), policy)
index.replace(&scope, changed, replacement)
if len(scope.collections) != 128 {
t.Fatal("replacement lost unrelated pending declarations")
}
local := index.localize(changed)
if _, found := local.collections["/introduced"]; !found || len(local.collections) != 1 {
t.Fatal("retired policy survived or introduced policy was not indexed")
}
index.replace(&scope, changed, collectionPolicyScope{})
if len(index.localize(changed).collections) != 0 || len(scope.collections) != 127 {
t.Fatal("descendant retirement did not update the table and index together")
}
if len(index.localize(Keys("group-065")).collections) != 1 {
t.Fatal("retirement changed another scope")
}
}
func TestInheritanceWideIndependentYamlCollectionScopes(t *testing.T) {
var body strings.Builder
body.WriteString("configkit:\n sequences:\n")
for index := range 64 {
fmt.Fprintf(&body, " - {destination: [groups, group-%03d, base, items], key: id, mode: append}\n", index)
}
body.WriteString("groups:\n")
for index := range 64 {
fmt.Fprintf(&body, " group-%03d:\n base: {items: [{id: shared, value: parent}]}\n child: {extends: base, items: [{id: shared, value: child}]}\n", index)
}
type item struct {
ID string `yaml:"id"`
Value string `yaml:"value"`
}
type definition struct {
Items []item `yaml:"items"`
}
type config struct {
Groups map[string]map[string]definition `yaml:"groups"`
}
loaded, err := testLoad[config](Bytes("wide-inheritance", []byte(body.String())))
if err != nil {
t.Fatal(err)
}
metadata := loaded.Metadata()
for name, group := range loaded.Config.Groups {
if len(group["base"].Items) != 1 || group["base"].Items[0].Value != "parent" || len(group["child"].Items) != 1 || group["child"].Items[0].Value != "child" {
t.Fatalf("independent scope %s composed incorrectly: %#v", name, group)
}
path := Keys("groups", name, "child", "items").Index(0).Key("value")
if info, exists := metadata.Lookup(path); !exists || !info.Explicit || info.Origin.Layer != LayerBase {
t.Fatalf("inherited destination lost source metadata at %s", path)
}
}
}

View File

@@ -7,28 +7,39 @@ import (
)
type inheritanceTemplate struct {
name string
entry document.Entry
path Path
parent int
hasParent bool
hasDirective bool
cancelled bool
directive document.Entry
name string
entry document.Entry
path Path
collectionPolicies collectionPolicyScope
parent int
hasParent bool
hasDirective bool
cancelled bool
directive document.Entry
}
type inheritanceGraph struct {
scope *document.Node
scopePath Path
nodes []inheritanceTemplate
edges []dependencyEdge
byName map[string]int
scope *document.Node
scopePath Path
collectionPolicies collectionPolicyScope
applicationPolicy sequencePolicy
nodes []inheritanceTemplate
edges []dependencyEdge
byName map[string]int
}
// analyzeInheritanceGraph retains only direct mapping siblings as graph nodes.
// It validates declarations separately from applying their templates, leaving
// nested scopes for the existing destination-local walk.
func analyzeInheritanceGraph(scope *document.Node, scopePath Path) (*inheritanceGraph, []Diagnostic) {
return analyzeInheritanceGraphWithPolicies(scope, scopePath, collectionPolicyScope{})
}
func analyzeInheritanceGraphWithPolicies(scope *document.Node, scopePath Path, policies collectionPolicyScope) (*inheritanceGraph, []Diagnostic) {
return analyzeInheritanceGraphWithCollectionRules(scope, scopePath, policies, sequencePolicy{})
}
func analyzeInheritanceGraphWithCollectionRules(scope *document.Node, scopePath Path, policies collectionPolicyScope, applicationPolicy sequencePolicy) (*inheritanceGraph, []Diagnostic) {
if scope == nil || scope.Kind != document.Mapping {
return nil, []Diagnostic{{
Code: CodeInheritanceInvalid, Severity: SeverityError, Path: scopePath,
@@ -36,8 +47,10 @@ func analyzeInheritanceGraph(scope *document.Node, scopePath Path) (*inheritance
}}
}
policyIndex := newInheritancePolicyIndex(policies)
graph := &inheritanceGraph{
scope: scope, scopePath: scopePath,
collectionPolicies: policyIndex.absoluteScope(scopePath), applicationPolicy: applicationPolicy,
byName: make(map[string]int, len(scope.Mapping)),
}
entries := make(map[string]document.Entry, len(scope.Mapping))
@@ -57,6 +70,7 @@ func analyzeInheritanceGraph(scope *document.Node, scopePath Path) (*inheritance
graph.byName[name] = index
graph.nodes = append(graph.nodes, inheritanceTemplate{
name: name, entry: entries[name], path: scopePath.Key(name),
collectionPolicies: policyIndex.localize(scopePath.Key(name)),
})
}
@@ -115,7 +129,15 @@ func (graph *inheritanceGraph) schedule() ([]int, []Diagnostic) {
}
func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
graph, diagnostics := analyzeInheritanceGraph(scope, scopePath)
return expandSiblingDefinitionsWithPolicies(scope, scopePath, limits, inputOrigins, collectionPolicyScope{}, budget...)
}
func expandSiblingDefinitionsWithPolicies(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, policies collectionPolicyScope, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
return expandSiblingDefinitionsWithCollectionRules(scope, scopePath, limits, inputOrigins, policies, sequencePolicy{}, budget...)
}
func expandSiblingDefinitionsWithCollectionRules(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, policies collectionPolicyScope, applicationPolicy sequencePolicy, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
graph, diagnostics := analyzeInheritanceGraphWithCollectionRules(scope, scopePath, policies, applicationPolicy)
if len(diagnostics) != 0 {
return nil, diagnostics
}
@@ -123,44 +145,56 @@ func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, l
if len(diagnostics) != 0 {
return nil, diagnostics
}
return applyInheritanceTemplates(graph, order, limits, inputOrigins, budget...)
result, diagnostics := applyInheritanceTemplates(graph, order, limits, inputOrigins, budget...)
if len(diagnostics) != 0 {
return nil, diagnostics
}
return result, nil
}
// applyInheritanceTemplates builds detached templates in the supplied
// dependency-ready order while retaining the existing origins, merge history,
// and node-accounting evidence. The root keeps its authored mapping order.
func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
resolved := make([]*document.Node, len(graph.nodes))
resolvedOrigins := make([]map[string]Origin, len(graph.nodes))
return applyInheritanceScopeTemplates(mappingInheritanceScope{graph: graph}, order, limits, inputOrigins, budget...)
}
func applyInheritanceScopeTemplates(adapter inheritanceScopeAdapter, order []int, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
templates := adapter.templates()
resolved := make([]*document.Node, len(templates))
resolvedOrigins := make([]map[string]Origin, len(templates))
resolvedPolicyTemplates := make([][]collectionPolicyScope, len(templates))
resolvedPolicies := make([]collectionPolicyScope, len(templates))
result := &inheritanceScope{}
counter := newRetainedNodeCounter(historyNodeLimit)
if len(budget) != 0 {
counter = budget[0]
}
retained := 1 // scope mapping node
for _, entry := range graph.scope.Mapping {
if entry.Value.Kind == document.Mapping {
retained := 1 // scope container node
for _, slot := range adapter.slots() {
if slot.hasTemplate || slot.value.Kind == document.Mapping {
continue
}
remaining := limits.nodes - retained - 1
if limits.nodes >= 0 && remaining < 0 {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(graph.scopePath.Key(entry.Key), entry.Value.Location)}
return nil, []Diagnostic{inheritanceBudgetDiagnostic(slot.path, slot.value.Location)}
}
count, within := inheritanceLowerBoundCount(entry.Value, remaining)
count, within := inheritanceLowerBoundCount(slot.value, remaining)
if !within {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(graph.scopePath.Key(entry.Key), entry.Value.Location)}
return nil, []Diagnostic{inheritanceBudgetDiagnostic(slot.path, slot.value.Location)}
}
retained += 1 + count
}
for _, index := range order {
node := graph.nodes[index]
node := templates[index]
templatePolicies := cloneCollectionPolicyScopeForInheritance(node.collectionPolicies)
payload := withoutInheritanceDirective(node.entry.Value)
operationStart, replacementStart := len(result.operations), len(result.replacements)
childOrigins := directInheritanceOrigins(payload, node.path, inputOrigins)
if node.hasDirective {
control := inheritanceControl{
path: node.path.Key(inheritanceKey),
path: node.directivePath,
locations: []Location{publicLocation(node.directive.Value.Location)},
cancelled: node.cancelled,
}
@@ -171,7 +205,7 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
})
}
if limits.nodes >= 0 {
remaining := limits.nodes - retained - 1 // the sibling key
remaining := limits.nodes - retained - adapter.slotOverhead()
within := inheritanceLowerBoundWithin(payload, remaining)
if node.hasParent {
within = inheritanceOverlayLowerBoundWithin(resolved[node.parent], payload, remaining)
@@ -183,14 +217,46 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
if node.hasParent {
parent := resolved[node.parent]
merged := document.Clone(parent)
parentPolicies := cloneInheritancePolicyTemplates(resolvedPolicyTemplates[node.parent])
effectivePolicies := append(cloneInheritancePolicyTemplates(parentPolicies), cloneCollectionPolicyScopeForInheritance(templatePolicies))
parentPolicyScope := projectInheritanceCollectionPolicies(resolvedPolicies[node.parent], node.path)
childPolicyScope := projectCurrentInheritanceCollectionPolicies(templatePolicies, node.path)
var replaced collectionReplacementPaths
if len(parentPolicyScope.collections) != 0 || len(childPolicyScope.collections) != 0 {
claims := compositionClaims{byPath: make(map[string][]compositionClaim)}
claims.append(payload, node.path, 0)
declarations := make(map[string][]sequenceDeclarationSpec, len(childPolicyScope.collections))
for key, entry := range childPolicyScope.collections {
declarations[key] = inheritanceDeclarationsAt(entry.destination, entry.policy)
}
parentPolicyScope, replaced = lowerCollectionPoliciesForClaims(parentPolicyScope, claims.ordered, declarations, adapter.applicationPolicy())
}
merged := inheritanceParentPayload(parent, node.identityFields)
historyParent := parent
if len(node.identityFields) != 0 {
// The retained parent snapshot is the inherited payload, before
// child fields are applied, with parent identity excluded.
historyParent = document.Clone(merged)
}
mergedPolicyScope, policyDiagnostics := mergeInheritanceCollectionPolicies(
parentPolicyScope, childPolicyScope, node.path, parent, payload, adapter.applicationPolicy(), replaced,
)
if len(policyDiagnostics) != 0 {
return nil, policyDiagnostics
}
transfer := inheritanceTransfer{
from: graph.nodes[node.parent].path, to: node.path,
reference: node.path.Key(inheritanceKey),
locations: []Location{publicLocation(node.directive.Value.Location)},
from: templates[node.parent].path, to: node.path,
reference: node.directivePath,
locations: []Location{publicLocation(node.directive.Value.Location)},
policyTemplates: parentPolicies, templatePolicies: cloneCollectionPolicyScopeForInheritance(templatePolicies),
projectedPolicies: projectInheritancePolicyTemplates(
[]collectionPolicyScope{resolvedPolicies[node.parent]}, node.path,
),
}
mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[node.parent], transfer.from, node.path, transfer)
mergeInheritanceOrigin(mergedOrigins, node.path, childOrigins[node.path.String()])
var sequenceContributions []sequenceHistoryContribution
var combinationDiagnostics []Diagnostic
mergeRecursiveMappings(merged, payload, node.path, new(mappingIndex), recursiveMergeHooks{
added: func(childPath Path, incoming *document.Node) {
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
@@ -207,21 +273,74 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
deleteOriginSubtree(mergedOrigins, childPath, existing)
copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
},
merge: func(childPath Path, existing, incoming *document.Node) (*document.Node, bool) {
resolution, _, _, active, issues := inheritanceCollectionResolution(
childPath, parentPolicyScope, childPolicyScope, adapter.applicationPolicy(), incoming,
)
if len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
return existing, false
}
if !active {
return nil, false
}
lowerValue := existing
combined, problems := combineSequence(lowerValue, incoming, resolution, sequenceCombinationContext{destination: childPath})
if len(problems) != 0 {
combinationDiagnostics = append(combinationDiagnostics, problems...)
return existing, false
}
policy := applyInheritanceSequencePolicy(
childPath, combined, resolution, parentPolicyScope, childPolicyScope,
)
mergedPolicyScope.set(childPath, policy)
applyInheritanceSequenceOrigins(
result, mergedOrigins, childOrigins, childPath, lowerValue, incoming, combined, resolution.operation,
)
history := sequenceHistoryContribution{path: childPath.clone(), mode: sequenceHistoryMapped}
if resolution.operation == sequenceModeReplace {
history.mode = sequenceHistoryReplace
if incoming != nil && incoming.Kind == document.Sequence {
history.length = len(incoming.Sequence)
}
} else {
history.items = append([]sequenceItemProjection(nil), combined.higherItems...)
history.replacements = append([]sequenceItemReplacement(nil), combined.replacements...)
}
sequenceContributions = append(sequenceContributions, history)
return combined.value, true
},
})
preserveInheritanceIdentityFields(merged, payload, node.identityFields)
if len(combinationDiagnostics) != 0 {
return nil, canonicalDiagnostics(combinationDiagnostics)
}
if problems := validateRegisteredInheritanceSequences(merged, node.path, adapter.applicationPolicy()); len(problems) != 0 {
return nil, problems
}
merged.Location = node.entry.Value.Location
resolved[index], resolvedOrigins[index] = merged, mergedOrigins
resolvedPolicies[index] = inheritancePolicyScopeAtPath(mergedPolicyScope, node.path)
result.transfers = append(result.transfers, transfer)
result.operations = append(result.operations, inheritanceOperation{
transfer: &transfer, parent: parent, effective: merged,
transfer: &transfer, parent: historyParent, effective: merged,
parentOrigins: resolvedOrigins[node.parent], effectiveOrigins: mergedOrigins,
parentPolicies: cloneInheritancePolicyTemplates(parentPolicies),
templatePolicies: cloneCollectionPolicyScopeForInheritance(templatePolicies),
effectivePolicies: cloneInheritancePolicyTemplates(effectivePolicies),
sequences: cloneSequenceHistoryContributions(sequenceContributions),
identityFields: cloneStrings(node.identityFields),
})
resolvedPolicyTemplates[index] = effectivePolicies
} else {
resolved[index] = document.Clone(payload)
resolvedOrigins[index] = childOrigins
resolvedPolicyTemplates[index] = []collectionPolicyScope{templatePolicies}
resolvedPolicies[index] = cloneCollectionPolicyScopeForInheritance(templatePolicies)
}
count, _ := inheritanceLowerBoundCount(resolved[index], limits.nodes)
retained += 1 + count
retained += adapter.slotOverhead() + count
for _, operation := range result.operations[operationStart:] {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
return nil, []Diagnostic{inheritanceHistoryBudgetDiagnostic(node.path, node.entry.Value.Location)}
@@ -234,26 +353,28 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
}
}
root := *graph.scope
root.Mapping = make([]document.Entry, len(graph.scope.Mapping))
result.origins = directInheritanceOrigins(&document.Node{Kind: document.Mapping, Location: graph.scope.Location}, graph.scopePath, inputOrigins)
for i, entry := range graph.scope.Mapping {
root.Mapping[i] = entry
if index, found := graph.byName[entry.Key]; found {
root.Mapping[i].Value = resolved[index]
root := adapter.rebuild(resolved)
result.collectionPolicies = collectionPolicyScope{}
result.origins = directInheritanceOrigins(&document.Node{Kind: root.Kind, Location: root.Location}, adapter.scopePath(), inputOrigins)
policyIndex := newInheritancePolicyIndex(adapter.collectionPolicies())
for _, slot := range adapter.slots() {
if slot.hasTemplate {
index := slot.template
projectInheritancePolicyRoots(&result.collectionPolicies, resolvedPolicies[index], templates[index].path)
for pointer, origin := range resolvedOrigins[index] {
result.origins[pointer] = origin
}
continue
}
root.Mapping[i].Value = document.Clone(entry.Value)
for pointer, origin := range directInheritanceOrigins(entry.Value, graph.scopePath.Key(entry.Key), inputOrigins) {
localPolicies := policyIndex.localize(slot.path)
projectInheritancePolicyRoots(&result.collectionPolicies, localPolicies, slot.path)
for pointer, origin := range directInheritanceOrigins(slot.value, slot.path, inputOrigins) {
result.origins[pointer] = origin
}
}
result.root = &root
result.root = root
if !inheritanceLowerBoundWithin(result.root, limits.nodes) {
return nil, []Diagnostic{inheritanceBudgetDiagnostic(graph.scopePath, graph.scope.Location)}
return nil, []Diagnostic{inheritanceBudgetDiagnostic(adapter.scopePath(), adapter.scope().Location)}
}
return result, nil
}

View File

@@ -110,6 +110,91 @@ beta:
}
}
func TestInheritanceTransportsReusableCollectionPolicyTemplates(t *testing.T) {
root := parseItemPolicyDocument(t, "templates", `base:
prompts:
- id: shared
children:
derived:
extends: nested-base
own: destination-local
nested-base:
inherited: source-value
first:
extends: base
second:
extends: base
`)
policyLocation := Location{SourceID: "base-resource", Line: 2, Column: 3}
nestedLocation := Location{SourceID: "nested-resource", Line: 5, Column: 11}
nested := collectionPolicy{identityFields: []string{"key"}, identityPresent: true, identityLocation: &nestedLocation}
recordScope := collectionPolicyScope{}
recordScope.set(Keys("children", "prompts"), nested)
basePolicy := collectionPolicy{identityFields: []string{"id"}, identityPresent: true, identityLocation: &policyLocation}
basePolicy.setRecordPolicyAt(0, collectionRecordPolicy{owner: newCollectionRecordPolicy().owner, scope: recordScope})
policies := collectionPolicyScope{}
policies.set(Keys("base", "prompts"), basePolicy)
got, diagnostics := expandSiblingDefinitionsWithPolicies(root, Path{}, defaultProjectionLimits(), nil, policies)
if len(diagnostics) != 0 {
t.Fatalf("expand sibling templates: %#v", diagnostics)
}
withoutPolicies, diagnostics := expandSiblingDefinitionsWithOrigins(root, Path{}, defaultProjectionLimits(), nil)
if len(diagnostics) != 0 {
t.Fatalf("expand sibling templates without policy: %#v", diagnostics)
}
if !reflect.DeepEqual(got.root, withoutPolicies.root) {
t.Fatalf("policy transport changed inherited values:\n got %#v\nwant %#v", got.root, withoutPolicies.root)
}
var transfers []inheritanceOperation
for _, operation := range got.operations {
if operation.transfer != nil {
transfers = append(transfers, operation)
}
}
if len(transfers) != 2 {
t.Fatalf("transfer operations = %d, want two destination transfers", len(transfers))
}
for index, destination := range []string{"/first", "/second"} {
operation := transfers[index]
if operation.transfer == nil || len(operation.transfer.projectedPolicies) != 2 {
t.Fatalf("transfer %d projected policies = %#v, want collection and record-local policy", index, operation.transfer)
}
projected := operation.transfer.projectedPolicies
if projected[0].path.String() != destination+"/prompts" || projected[1].path.String() != destination+"/prompts/0/children/prompts" {
t.Fatalf("transfer %d policy paths = %q, %q", index, projected[0].path, projected[1].path)
}
if got := projected[0].policy.identityLocation; got == nil || *got != policyLocation {
t.Fatalf("transfer %d collection declaration location = %#v", index, got)
}
if got := projected[1].policy.identityLocation; got == nil || *got != nestedLocation {
t.Fatalf("transfer %d record-local declaration location = %#v", index, got)
}
if len(operation.templatePolicies.collections) != 0 {
t.Fatalf("transfer %d unexpectedly copied parent YAML policy onto child template", index)
}
}
first, second := transfers[0].transfer.projectedPolicies[0].policy, transfers[1].transfer.projectedPolicies[0].policy
if first.recordOrder[0] == second.recordOrder[0] {
t.Fatal("destination policy templates share a mutable record owner")
}
first.identityFields[0] = "changed"
if second.identityFields[0] != "id" {
t.Fatalf("mutating one destination changed another: %v", second.identityFields)
}
if basePolicy.identityFields[0] != "id" {
t.Fatalf("mutating a destination changed the source template: %v", basePolicy.identityFields)
}
firstValue, found := documentAtPath(got.root.Mapping[1].Value, Keys("prompts").Index(0).Key("children").Key("derived").Key("extends"))
if !found || firstValue.Text != "nested-base" {
t.Fatalf("nested template instruction = %#v, found %t; want original instruction retained", firstValue, found)
}
}
func TestInheritanceExpansionInterleavesUnequalBranches(t *testing.T) {
root := parseCompositionDocument(t, "branches", `a-grandchild:
extends: z-child

View File

@@ -0,0 +1,198 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
// inheritanceScopeAdapter keeps relationship execution independent from the
// container that owns the records. Adapters are private preparation machinery;
// public loading continues to use mapping scopes until record inheritance is
// activated separately.
type inheritanceScopeAdapter interface {
scope() *document.Node
scopePath() Path
collectionPolicies() collectionPolicyScope
applicationPolicy() sequencePolicy
templates() []inheritanceExecutionTemplate
slots() []inheritanceScopeSlot
slotOverhead() int
rebuild(resolved []*document.Node) *document.Node
}
type inheritanceExecutionTemplate struct {
entry document.Entry
path Path
collectionPolicies collectionPolicyScope
parent int
hasParent bool
hasDirective bool
cancelled bool
directive document.Entry
directivePath Path
identityFields []string
}
type inheritanceScopeSlot struct {
value *document.Node
path Path
template int
hasTemplate bool
}
type mappingInheritanceScope struct{ graph *inheritanceGraph }
func (scope mappingInheritanceScope) scope() *document.Node { return scope.graph.scope }
func (scope mappingInheritanceScope) scopePath() Path { return scope.graph.scopePath }
func (scope mappingInheritanceScope) collectionPolicies() collectionPolicyScope {
return scope.graph.collectionPolicies
}
func (scope mappingInheritanceScope) applicationPolicy() sequencePolicy {
return scope.graph.applicationPolicy
}
func (scope mappingInheritanceScope) slotOverhead() int { return 1 }
func (scope mappingInheritanceScope) templates() []inheritanceExecutionTemplate {
result := make([]inheritanceExecutionTemplate, len(scope.graph.nodes))
for index, node := range scope.graph.nodes {
var directive document.Entry
if node.hasDirective {
directive = node.directive
}
result[index] = inheritanceExecutionTemplate{
entry: node.entry, path: node.path,
collectionPolicies: node.collectionPolicies, parent: node.parent,
hasParent: node.hasParent, hasDirective: node.hasDirective,
cancelled: node.cancelled, directive: directive,
directivePath: node.path.Key(inheritanceKey),
}
}
return result
}
func (scope mappingInheritanceScope) slots() []inheritanceScopeSlot {
result := make([]inheritanceScopeSlot, len(scope.graph.scope.Mapping))
for index, entry := range scope.graph.scope.Mapping {
slot := inheritanceScopeSlot{
value: entry.Value, path: scope.graph.scopePath.Key(entry.Key),
}
if template, ok := scope.graph.byName[entry.Key]; ok {
slot.template, slot.hasTemplate = template, true
}
result[index] = slot
}
return result
}
func (scope mappingInheritanceScope) rebuild(resolved []*document.Node) *document.Node {
root := *scope.graph.scope
root.Mapping = make([]document.Entry, len(scope.graph.scope.Mapping))
for index, entry := range scope.graph.scope.Mapping {
root.Mapping[index] = entry
if template, ok := scope.graph.byName[entry.Key]; ok {
root.Mapping[index].Value = resolved[template]
} else {
root.Mapping[index].Value = document.Clone(entry.Value)
}
}
return &root
}
type identifiedSequenceInheritanceScope struct {
graph *identifiedInheritanceGraph
policies collectionPolicyScope
application sequencePolicy
}
func (scope identifiedSequenceInheritanceScope) scope() *document.Node { return scope.graph.scope }
func (scope identifiedSequenceInheritanceScope) scopePath() Path { return scope.graph.scopePath }
func (scope identifiedSequenceInheritanceScope) collectionPolicies() collectionPolicyScope {
return scope.policies
}
func (scope identifiedSequenceInheritanceScope) applicationPolicy() sequencePolicy {
return scope.application
}
func (scope identifiedSequenceInheritanceScope) slotOverhead() int { return 0 }
func (scope identifiedSequenceInheritanceScope) templates() []inheritanceExecutionTemplate {
result := make([]inheritanceExecutionTemplate, len(scope.graph.nodes))
policyIndex := newInheritancePolicyIndex(scope.policies)
for index, node := range scope.graph.nodes {
result[index] = inheritanceExecutionTemplate{
entry: document.Entry{Value: node.item}, path: node.path,
collectionPolicies: policyIndex.localize(node.path),
parent: node.parent, hasParent: node.hasParent,
hasDirective: node.directive != nil, cancelled: node.cancelled,
directive: document.Entry{Value: node.directive}, directivePath: node.directivePath,
identityFields: scope.graph.fields,
}
}
return result
}
func (scope identifiedSequenceInheritanceScope) slots() []inheritanceScopeSlot {
result := make([]inheritanceScopeSlot, len(scope.graph.nodes))
for index, node := range scope.graph.nodes {
result[index] = inheritanceScopeSlot{
value: node.item, path: node.path, template: index, hasTemplate: true,
}
}
return result
}
func (scope identifiedSequenceInheritanceScope) rebuild(resolved []*document.Node) *document.Node {
root := *scope.graph.scope
root.Sequence = make([]*document.Node, len(scope.graph.nodes))
for index := range root.Sequence {
root.Sequence[index] = resolved[index]
}
return &root
}
func preserveInheritanceIdentityFields(merged, child *document.Node, fields []string) {
if len(fields) == 0 || merged == nil || merged.Kind != document.Mapping || child == nil || child.Kind != document.Mapping {
return
}
for _, field := range fields {
for _, entry := range child.Mapping {
if entry.Key != field {
continue
}
for index := range merged.Mapping {
if merged.Mapping[index].Key == field {
merged.Mapping[index] = document.Entry{Key: field, Value: document.Clone(entry.Value)}
break
}
}
break
}
}
}
func inheritanceParentPayload(parent *document.Node, identityFields []string) *document.Node {
cloned := document.Clone(parent)
if len(identityFields) == 0 || cloned == nil || cloned.Kind != document.Mapping {
return cloned
}
excluded := make(map[string]struct{}, len(identityFields))
for _, field := range identityFields {
excluded[field] = struct{}{}
}
kept := cloned.Mapping[:0]
for _, entry := range cloned.Mapping {
if _, omit := excluded[entry.Key]; !omit {
kept = append(kept, entry)
}
}
cloned.Mapping = kept
return cloned
}
func applyIdentifiedInheritanceTemplates(
graph *identifiedInheritanceGraph,
limits projectionLimits,
inputOrigins map[string]Origin,
policies collectionPolicyScope,
application sequencePolicy,
budget ...*retainedNodeCounter,
) (*inheritanceScope, []Diagnostic) {
order, diagnostics := graph.schedule()
if len(diagnostics) != 0 {
return nil, diagnostics
}
adapter := identifiedSequenceInheritanceScope{
graph: graph, policies: policies, application: application,
}
return applyInheritanceScopeTemplates(adapter, order, limits, inputOrigins, budget...)
}

View File

@@ -8,14 +8,17 @@ import (
)
type inheritanceWalker struct {
nodes int // logical non-directive nodes in the entire working tree
origins map[string]Origin
limits projectionLimits
retained *retainedNodeCounter
transfers []inheritanceTransfer
controls []inheritanceControl
replacements []inheritanceReplacement
operations []inheritanceOperation
nodes int // logical non-directive nodes in the entire working tree
origins map[string]Origin
collectionPolicies collectionPolicyScope
policyIndex *inheritancePolicyIndex
applicationPolicy sequencePolicy
limits projectionLimits
retained *retainedNodeCounter
transfers []inheritanceTransfer
controls []inheritanceControl
replacements []inheritanceReplacement
operations []inheritanceOperation
}
// expandInheritanceTree resolves each destination scope after its enclosing
@@ -25,10 +28,26 @@ func expandInheritanceTree(input *document.Node, limits projectionLimits) (*inhe
}
func expandInheritanceWithOrigins(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
return expandInheritanceWithBudgets(input, inputOrigins, limits, historyNodeLimit)
return expandInheritanceWithPolicies(input, inputOrigins, collectionPolicyScope{}, limits)
}
func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits, historyLimit int) (*inheritanceScope, []Diagnostic) {
return expandInheritanceWithPolicyBudgets(input, inputOrigins, collectionPolicyScope{}, limits, historyLimit)
}
func expandInheritanceWithPolicies(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
return expandInheritanceWithCollectionRules(input, inputOrigins, policies, sequencePolicy{}, limits)
}
func expandInheritanceWithPolicyBudgets(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, limits projectionLimits, historyLimit int) (*inheritanceScope, []Diagnostic) {
return expandInheritanceWithCollectionRuleBudgets(input, inputOrigins, policies, sequencePolicy{}, limits, historyLimit)
}
func expandInheritanceWithCollectionRules(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, applicationPolicy sequencePolicy, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
return expandInheritanceWithCollectionRuleBudgets(input, inputOrigins, policies, applicationPolicy, limits, historyNodeLimit)
}
func expandInheritanceWithCollectionRuleBudgets(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, applicationPolicy sequencePolicy, limits projectionLimits, historyLimit int) (*inheritanceScope, []Diagnostic) {
if input == nil || input.Kind != document.Mapping {
return nil, []Diagnostic{{Code: CodeInheritanceInvalid, Severity: SeverityError,
Message: "inheritance requires a mapping document root"}}
@@ -44,8 +63,11 @@ func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]
}
walker := inheritanceWalker{
nodes: nodes, limits: limits, origins: origins,
retained: newRetainedNodeCounter(historyLimit),
collectionPolicies: cloneCollectionPolicyScopeForInheritance(policies),
applicationPolicy: applicationPolicy,
retained: newRetainedNodeCounter(historyLimit),
}
walker.policyIndex = newInheritancePolicyIndex(walker.collectionPolicies)
if problem := walker.visit(&root, Path{}, true); problem != nil {
return nil, []Diagnostic{*problem}
}
@@ -53,8 +75,10 @@ func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]
return nil, []Diagnostic{*problem}
}
return &inheritanceScope{
root: root, origins: walker.origins, transfers: walker.transfers,
controls: walker.controls, replacements: walker.replacements, operations: walker.operations,
root: root, origins: walker.origins,
collectionPolicies: cloneCollectionPolicyScopeForInheritance(walker.collectionPolicies),
transfers: walker.transfers,
controls: walker.controls, replacements: walker.replacements, operations: walker.operations,
}, nil
}
@@ -90,11 +114,14 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool
if limits.nodes >= 0 {
limits.nodes -= w.nodes - before
}
resolved, diagnostics := expandSiblingDefinitionsWithOrigins(*node, path, limits, w.origins, w.retained)
resolved, diagnostics := expandSiblingDefinitionsWithCollectionRules(
*node, path, limits, w.origins, w.policyIndex.absoluteScope(path), w.applicationPolicy, w.retained,
)
if len(diagnostics) != 0 {
return &diagnostics[0]
}
*node = resolved.root
w.policyIndex.replace(&w.collectionPolicies, path, resolved.collectionPolicies)
deleteOriginSubtree(w.origins, path, original)
for pointer, origin := range resolved.origins {
w.origins[pointer] = origin
@@ -126,6 +153,11 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool
}
}
case document.Sequence:
if sequenceHasIdentifiedInheritanceReferences(*node) {
if problem := w.expandIdentifiedSequence(node, path); problem != nil {
return problem
}
}
working := **node
working.Sequence = slices.Clone(working.Sequence)
*node = &working
@@ -139,6 +171,53 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool
return nil
}
func (w *inheritanceWalker) expandIdentifiedSequence(node **document.Node, path Path) *Diagnostic {
policy, hasPolicy := w.collectionPolicies.at(path)
applicationFields, _ := w.applicationPolicy.identityFieldsAt(path)
var lower *collectionPolicy
if hasPolicy {
lower = &policy
}
resolution, policyDiagnostics := resolveCollectionPolicy(path, nil, applicationFields, lower)
if len(policyDiagnostics) != 0 {
return &policyDiagnostics[0]
}
// Without an identity rule, retain the existing diagnostic for a reference
// that has no enclosing sibling scope. A null cancellation needs no lookup.
if len(resolution.effectiveIdentityFields) == 0 {
return nil
}
original := *node
graph, diagnostics := analyzeIdentifiedInheritanceGraph(*node, path, resolution.effectiveIdentityFields)
if len(diagnostics) != 0 {
return &diagnostics[0]
}
before, _ := inheritanceLowerBoundCount(*node, -1)
limits := w.limits
if limits.nodes >= 0 {
limits.nodes -= w.nodes - before
}
resolved, diagnostics := applyIdentifiedInheritanceTemplates(
graph, limits, w.origins, w.policyIndex.absoluteScope(path), w.applicationPolicy, w.retained,
)
if len(diagnostics) != 0 {
return &diagnostics[0]
}
*node = resolved.root
w.policyIndex.replace(&w.collectionPolicies, path, resolved.collectionPolicies)
deleteOriginSubtree(w.origins, path, original)
for pointer, origin := range resolved.origins {
w.origins[pointer] = origin
}
w.transfers = append(w.transfers, resolved.transfers...)
w.controls = append(w.controls, resolved.controls...)
w.replacements = append(w.replacements, resolved.replacements...)
w.operations = append(w.operations, resolved.operations...)
after, _ := inheritanceLowerBoundCount(*node, -1)
w.nodes += after - before
return nil
}
func scopeHasInheritanceDefinitions(scope *document.Node) bool {
for _, entry := range scope.Mapping {
if entry.Value.Kind != document.Mapping {

View File

@@ -0,0 +1,471 @@
package engine
import (
"fmt"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml"
)
func parseItemPolicyDocument(t *testing.T, sourceID, input string) *document.Node {
t.Helper()
root, issues := yamladapter.Parse(strings.NewReader(input), sourceID, yamladapter.Limits{
Depth: maxDocumentDepth, Nodes: maxDocumentNodes,
})
if len(issues) != 0 {
t.Fatalf("parse document: %#v", issues)
}
return root
}
func TestPendingItemCompositionKeepsLocalDeclarationsAndHelperIdentity(t *testing.T) {
t.Parallel()
declaration := sequenceDeclarationSpec{
destination: Keys("prompts"), identityFields: []string{"id"}, identityPresent: true,
}
itemScope := acquisitionScope{contributions: []acquiredContribution{
{
document: parseItemPolicyDocument(t, "item", "name: authored\nprompts:\n - id: authored\n"),
sequenceDeclarations: []sequenceDeclarationSpec{declaration},
},
{
document: parseItemPolicyDocument(t, "helper", "id: supplied\nprompts:\n - id: helper\n"),
},
}}
outer := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "item"},
payload: parseItemPolicyDocument(t, "outer", "{}\n"),
destination: Keys("items"), pendingItem: &itemScope,
}}}
composed, diagnostics := composeAcquiredScopeWithPolicy(
&outer, collectionPolicyScope{}, sequencePolicy{}, newImportInclusionGraph(), nil, nil, nil, &importAccounting{},
)
if len(diagnostics) != 0 {
t.Fatalf("compose pending item scope: %#v", diagnostics)
}
item := outer.contributions[0].payload
if got, ok := documentAtPath(item, Keys("id")); !ok || got == nil || got.Text != "supplied" {
t.Fatalf("helper-supplied record identity = %#v", got)
}
if got, ok := documentAtPath(item, Keys("prompts")); !ok || got == nil || got.Kind != document.Sequence || len(got.Sequence) != 2 {
t.Fatalf("composed prompts = %#v, want two identified entries", got)
}
itemPolicy, ok := composed.collectionPolicies.at(Keys("items"))
if !ok {
t.Fatalf("projected item collection policy missing")
}
owned, ok := itemPolicy.recordPolicyAt(0)
if !ok {
t.Fatalf("projected item owner missing: %#v", itemPolicy)
}
if policy, ok := owned.scope.at(Keys("prompts")); !ok || !policy.identityPresent || len(policy.identityFields) != 1 || policy.identityFields[0] != "id" {
t.Fatalf("item-owned prompt policy = %#v, %v", policy, ok)
}
if _, ok := composed.collectionPolicies.at(Keys("prompts")); ok {
t.Fatal("item-local declaration escaped its owning record")
}
}
func TestPendingItemCompositionProjectsNestedIdentityDiagnostics(t *testing.T) {
t.Parallel()
childScope := acquisitionScope{contributions: []acquiredContribution{
{document: parseItemPolicyDocument(t, "child", "prompts:\n - id: same\n"), sequenceDeclarations: []sequenceDeclarationSpec{{
destination: Keys("prompts"), identityFields: []string{"id"}, identityPresent: true,
}}},
{document: parseItemPolicyDocument(t, "child-helper", "prompts:\n - id: same\n")},
}}
itemScope := acquisitionScope{contributions: []acquiredContribution{{
document: parseItemPolicyDocument(t, "item", "name: parent\n"),
}, {
source: SourceInfo{ID: "child"}, payload: parseItemPolicyDocument(t, "child", "{}\n"),
destination: Keys("children"), pendingItem: &childScope,
}}}
outer := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "item"}, payload: parseItemPolicyDocument(t, "outer", "{}\n"),
destination: Keys("items"), pendingItem: &itemScope,
}}}
traversal := importTraversal{accounting: &importAccounting{}, limits: defaultImportLimits()}
diagnostics := traversal.composePendingItems(&outer)
if len(diagnostics) != 1 {
t.Fatalf("diagnostics = %#v, want one conflict", diagnostics)
}
if diagnostics[0].Code != CodeComposeConflict || diagnostics[0].Path.String() != "/items/0/children/0/prompts" {
t.Fatalf("diagnostic = %#v, want projected collection path", diagnostics[0])
}
if strings.Contains(fmt.Sprint(diagnostics[0]), "same") {
t.Fatalf("diagnostic disclosed identity value: %v", diagnostics[0])
}
}
func TestPendingItemCompositionAppliesOuterApplicationIdentityConstraints(t *testing.T) {
t.Parallel()
itemScope := acquisitionScope{contributions: []acquiredContribution{{
document: parseItemPolicyDocument(t, "item", "prompts:\n - text: missing-id\n"),
}}}
outer := acquisitionScope{contributions: []acquiredContribution{{
source: SourceInfo{ID: "item"}, payload: parseItemPolicyDocument(t, "outer", "{}\n"),
destination: Keys("items"), pendingItem: &itemScope,
}}}
policy, diagnostics := compileSequenceIdentityPolicy([]sequenceIdentityRegistration{{
selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"},
}})
if len(diagnostics) != 0 {
t.Fatalf("compile application identity: %#v", diagnostics)
}
_, diagnostics = composeAcquiredScopeWithPolicy(&outer, collectionPolicyScope{}, policy,
newImportInclusionGraph(), nil, nil, nil, &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeComposeInvalid || diagnostics[0].Path.String() != "/items/0/id" {
t.Fatalf("application identity diagnostics = %#v", diagnostics)
}
}
func TestReplacingOuterCollectionRetiresDiscardedItemIdentity(t *testing.T) {
t.Parallel()
outerPath := Keys("items")
nestedPath := Keys("items").Index(0).Key("prompts")
lower := collectionPolicyScope{}
outerPolicy := collectionPolicy{}
outerPolicy.setRecordPolicy(newCollectionRecordPolicy())
lower.set(outerPath, outerPolicy)
lower.set(nestedPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
declarations := []sequenceDeclarationSpec{{
destination: outerPath, mode: sequenceModeReplace, modePresent: true,
}}
input := parseItemPolicyDocument(t, "replacement", "items:\n - prompts:\n - text: new\n")
composed, diagnostics := composeDocumentsWithCollectionPolicy(sequencePolicy{}, lower, declarations, input)
if len(diagnostics) != 0 {
t.Fatalf("compose replacement: %#v", diagnostics)
}
policy, ok := composed.collectionPolicies.at(nestedPath)
if !ok || policy.identityPresent || len(policy.recordOrder) != 0 {
t.Fatalf("discarded nested owner policy = %#v, %v", policy, ok)
}
}
func TestNestedItemPolicyProjectionUsesStableOutputIndexes(t *testing.T) {
t.Parallel()
itemsPath := Keys("items")
outerPolicy := collectionPolicy{}
retainedScope := collectionPolicyScope{}
retainedScope.set(Keys("retained"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
outerPolicy.setRecordPolicy(collectionRecordPolicy{owner: newCollectionRecordPolicy().owner, scope: retainedScope})
composed := &composedDocument{collectionPolicies: collectionPolicyScope{}}
composed.collectionPolicies.set(itemsPath, outerPolicy)
firstScope := collectionPolicyScope{}
firstScope.set(Keys("prompts"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
secondScope := collectionPolicyScope{}
secondScope.set(Keys("prompts"), collectionPolicy{identityFields: []string{"slug"}, identityPresent: true})
scope := acquisitionScope{contributions: []acquiredContribution{
{document: parseItemPolicyDocument(t, "first", "items:\n - id: first\n"), destination: itemsPath, nested: &composedDocument{collectionPolicies: firstScope}},
{document: parseItemPolicyDocument(t, "second", "items:\n - id: second\n"), destination: itemsPath, nested: &composedDocument{collectionPolicies: secondScope}},
}}
projectNestedCollectionPolicies(composed, scope, nil)
policy, ok := composed.collectionPolicies.at(itemsPath)
if !ok || len(policy.recordOrder) != 3 {
t.Fatalf("record-owned policies = %#v, %v; want retained plus two item scopes", policy, ok)
}
for index, wantField := range []string{"id", "id", "slug"} {
record, ok := policy.recordPolicyAt(index)
if !ok {
t.Fatalf("record policy %d missing", index)
}
path := Keys("retained")
if index != 0 {
path = Keys("prompts")
}
nested, ok := record.scope.at(path)
if !ok || !nested.identityPresent || len(nested.identityFields) != 1 || nested.identityFields[0] != wantField {
t.Errorf("record policy %d = %#v, %v; want %q", index, nested, ok, wantField)
}
}
}
func TestItemPolicyProjectionCountsContributionsWithoutLocalPolicy(t *testing.T) {
t.Parallel()
items := Keys("items")
for _, assignments := range []map[int]int{nil, {1: 2, 3: 4}, {1: 2}, {3: 4}} {
t.Run(fmt.Sprint(assignments), func(t *testing.T) {
local := collectionPolicyScope{}
local.set(Keys("steps"), collectionPolicy{identityFields: []string{"name"}, identityPresent: true})
scope := acquisitionScope{contributions: []acquiredContribution{
{document: parseItemPolicyDocument(t, "inline", "items: [{id: inline-a}, {id: inline-b}]\n")},
{document: parseItemPolicyDocument(t, "first", "items: [{id: first}]\n"), destination: items, nested: &composedDocument{collectionPolicies: local}},
{document: parseItemPolicyDocument(t, "gap", "items: [{id: gap}]\n"), destination: items},
{document: parseItemPolicyDocument(t, "second", "items: [{id: second}]\n"), destination: items, nested: &composedDocument{collectionPolicies: local}},
}}
ranges := make(map[sequenceRangeKey]sequenceRange)
for sourceIndex, output := range assignments {
ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: items.String()}] = sequenceRange{start: output, end: output + 1}
}
composed := &composedDocument{}
projectNestedCollectionPolicies(composed, scope, ranges)
policy, _ := composed.collectionPolicies.at(items)
if len(policy.recordOrder) != 5 {
t.Fatalf("record policies = %d, want 5 positions", len(policy.recordOrder))
}
for index := range 5 {
record, _ := policy.recordPolicyAt(index)
_, found := record.scope.at(Keys("steps"))
if found != (index == 2 || index == 4) {
t.Errorf("local policy at item %d = %t", index, found)
}
}
})
}
}
func TestItemPolicyProjectionDetachesRepeatedScopes(t *testing.T) {
t.Parallel()
local := collectionPolicyScope{}
local.set(Keys("steps"), collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
scope := acquisitionScope{}
ranges := make(map[sequenceRangeKey]sequenceRange)
for index, destination := range []string{"items", "other", "items"} {
path := Keys(destination)
scope.contributions = append(scope.contributions, acquiredContribution{
document: parseItemPolicyDocument(t, "item", destination+": [{}]\n"), destination: path,
nested: &composedDocument{collectionPolicies: local},
})
output := 0
if index == 2 {
output = 1
}
ranges[sequenceRangeKey{sourceIndex: index, path: path.String()}] = sequenceRange{start: output, end: output + 1}
}
first, second := &composedDocument{}, &composedDocument{}
projectNestedCollectionPolicies(first, scope, ranges)
projectNestedCollectionPolicies(second, scope, ranges)
policy := first.collectionPolicies.collections[Keys("items").String()].policy
if len(policy.recordOrder) != 2 || policy.recordOrder[0] == policy.recordOrder[1] {
t.Fatal("separately placed records did not retain distinct owners")
}
other := second.collectionPolicies.collections[Keys("items").String()].policy
if policy.recordOrder[0] == other.recordOrder[0] {
t.Fatal("independent projections reused a record owner")
}
// Mutate stored state directly: a detached query result would conceal an
// accidental alias between incrementally constructed record scopes.
steps := Keys("steps").String()
firstRecord := policy.recordPolicies[policy.recordOrder[0]]
firstRecord.collections[steps].policy.identityFields[0] = "changed"
for _, check := range []struct {
composed *composedDocument
path Path
index int
}{
{first, Keys("items"), 1}, {first, Keys("other"), 0},
{second, Keys("items"), 0}, {second, Keys("items"), 1}, {second, Keys("other"), 0},
} {
outer, _ := check.composed.collectionPolicies.at(check.path)
record, found := outer.recordPolicyAt(check.index)
identity, present := record.scope.at(Keys("steps"))
if !found || !present || !reflect.DeepEqual(identity.identityFields, []string{"id"}) {
t.Fatalf("mutation leaked into %s item %d: %#v", check.path, check.index, identity)
}
}
if local.collections[steps].policy.identityFields[0] != "id" {
t.Fatal("projected state mutated the incoming scope")
}
local.collections[steps].policy.identityFields[0] = "source-changed"
if other.recordPolicies[other.recordOrder[0]].collections[steps].policy.identityFields[0] != "id" {
t.Fatal("incoming scope remained aliased to projected state")
}
}
func TestOverlaySequenceTransfersIncomingItemPolicyOwnership(t *testing.T) {
path := Keys("records")
registered := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("records")}, identityFields: []string{"id"}}})
makePolicies := func(childKey string) collectionPolicyScope {
local := collectionPolicyScope{}
local.set(Keys("steps"), collectionPolicy{
identityFields: []string{childKey}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
})
record := newCollectionRecordPolicy()
record.scope = local
outer := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
outer.setRecordPolicy(record)
scope := collectionPolicyScope{}
scope.set(path, outer)
return scope
}
checkPolicy := func(t *testing.T, scope collectionPolicyScope, index int, key string) {
t.Helper()
outer, ok := scope.at(path)
if !ok {
t.Fatal("outer collection policy missing")
}
record, ok := outer.recordPolicyAt(index)
if !ok {
t.Fatalf("record policy %d missing", index)
}
local, ok := record.scope.at(Keys("steps"))
if !ok || !local.identityPresent || !reflect.DeepEqual(local.identityFields, []string{key}) || !local.explicitModePresent || local.explicitMode != sequenceModeAppend {
t.Fatalf("record %d nested policy = %#v, %v; want key %q and append", index, local, ok, key)
}
}
tests := []struct {
name string
lower string
higher string
declarations []sequenceDeclarationSpec
wantRecords int
}{
{
name: "upserting the same outer identity replaces its local policy",
lower: "records: [{id: same, steps: [{id: old}]}]\n",
higher: "records: [{id: same, steps: [{slug: new}]}]\n",
wantRecords: 1,
},
{
name: "retained and appended outer records keep independent policies",
lower: "records: [{id: lower, steps: [{id: old}]}]\n",
higher: "records: [{id: higher, steps: [{slug: new}]}]\n",
wantRecords: 2,
},
{
name: "whole sequence replacement transfers new local policy",
lower: "records: [{id: same, steps: [{id: old}]}]\n",
higher: "records: [{id: same, steps: [{slug: new}]}]\n",
declarations: []sequenceDeclarationSpec{{
destination: path, mode: sequenceModeReplace, modePresent: true,
}},
wantRecords: 1,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
lowerRoot := parseItemPolicyDocument(t, "lower", test.lower)
incomingRoot := parseItemPolicyDocument(t, "higher", test.higher)
assembly := newAssemblyAccumulator(&composedDocument{
root: lowerRoot, collectionPolicies: makePolicies("id"), origins: make(map[string]Origin),
})
if diagnostics := assembly.applyOverlayWithCollectionPolicy(
incomingRoot, nil, makePolicies("slug"), nil, LayerBase, registered, test.declarations,
); len(diagnostics) != 0 {
t.Fatalf("overlay diagnostics = %#v", diagnostics)
}
outer, ok := assembly.current.collectionPolicies.at(path)
if !ok || len(outer.recordOrder) != test.wantRecords {
t.Fatalf("outer record policies = %#v, %v", outer, ok)
}
if test.wantRecords == 2 {
checkPolicy(t, assembly.current.collectionPolicies, 0, "id")
checkPolicy(t, assembly.current.collectionPolicies, 1, "slug")
} else {
checkPolicy(t, assembly.current.collectionPolicies, 0, "slug")
}
})
}
}
func TestAncestorReplacementRequiresIdentityWhenRetainedModeReturns(t *testing.T) {
path := Keys("catalog", "prompts")
newAssembly := func(t *testing.T) *assemblyAccumulator {
t.Helper()
policy := collectionPolicy{
identityFields: []string{"id"}, identityPresent: true,
explicitMode: sequenceModeAppend, explicitModePresent: true,
}
scope := collectionPolicyScope{}
scope.set(path, policy)
return newAssemblyAccumulator(&composedDocument{
root: parseItemPolicyDocument(t, "base", "catalog:\n prompts: [{id: old}]\n"),
collectionPolicies: scope, origins: make(map[string]Origin),
})
}
removeAncestor := func(t *testing.T, assembly *assemblyAccumulator) {
t.Helper()
if diagnostics := assembly.applyOverlayWithCollectionPolicy(
parseItemPolicyDocument(t, "remove", "catalog: null\n"), nil, collectionPolicyScope{}, nil,
LayerBase, sequencePolicy{}, nil,
); len(diagnostics) != 0 {
t.Fatalf("ancestor removal diagnostics = %#v", diagnostics)
}
}
t.Run("retained append mode requires a renewed identity", func(t *testing.T) {
assembly := newAssembly(t)
removeAncestor(t, assembly)
policy, ok := assembly.current.collectionPolicies.at(path)
if !ok || policy.identityPresent || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
t.Fatalf("retired collection policy = %#v, %v", policy, ok)
}
diagnostics := assembly.applyOverlayWithCollectionPolicy(
parseItemPolicyDocument(t, "reintroduction", "catalog:\n prompts: [{id: restored, name: restored}]\n"), nil,
collectionPolicyScope{}, nil, LayerBase, sequencePolicy{}, nil,
)
if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid {
t.Fatalf("append without renewed identity diagnostics = %#v", diagnostics)
}
})
t.Run("a renewed key restores the retained append mode", func(t *testing.T) {
assembly := newAssembly(t)
removeAncestor(t, assembly)
declaration := sequenceDeclarationSpec{
destination: path, identityFields: []string{"name"}, identityPresent: true,
}
diagnostics := assembly.applyOverlayWithCollectionPolicy(
parseItemPolicyDocument(t, "keyed", "catalog:\n prompts: [{name: next}]\n"), nil,
collectionPolicyScope{}, nil, LayerBase, sequencePolicy{}, []sequenceDeclarationSpec{declaration},
)
if len(diagnostics) != 0 {
t.Fatalf("renewed identity diagnostics = %#v", diagnostics)
}
policy, ok := assembly.current.collectionPolicies.at(path)
if !ok || !policy.identityPresent || !reflect.DeepEqual(policy.identityFields, []string{"name"}) || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
t.Fatalf("renewed collection policy = %#v, %v", policy, ok)
}
})
}
func TestItemSourceVersionProcessingConsumesDeclarationsBeforeComposition(t *testing.T) {
t.Parallel()
root := parseItemPolicyDocument(t, "item", `version: v1
configkit:
sequences:
- destination: [prompts]
key: id
prompts:
- id: example
`)
rule := &VersionRule{Path: Keys("version"), Check: func(version Version, _ *Validation) error {
if !version.Present || version.Text != "v1" {
t.Errorf("version = %#v, want v1", version)
}
return nil
}}
processed := processSourceVersionWithDirectiveContext(
root, rule, sourceVersionContribution, Keys("items").Index(0), sourceVersionModeRequired, rootDirectiveContextSource,
)
if len(processed.diagnostics) != 0 {
t.Fatalf("process item source version: %#v", processed.diagnostics)
}
if len(processed.sequences) != 1 || !processed.sequences[0].identityPresent {
t.Fatalf("extracted item declarations = %#v", processed.sequences)
}
if _, ok := documentAtPath(processed.payload, Keys("version")); ok {
t.Fatalf("consumed version header remains in item payload: %#v", processed.payload)
}
if _, ok := documentAtPath(processed.payload, Keys("configkit")); ok {
t.Fatalf("consumed controls remain in item payload: %#v", processed.payload)
}
}

View File

@@ -17,12 +17,14 @@ type Definition[T any] struct {
Version *VersionRule
Secret func(Path) bool
sequenceRegistrations []sequenceRegistration
sequenceIdentityRegistrations []sequenceIdentityRegistration
}
// Options contains the application definition and per-load input selections.
// Imports enables explicit import declarations; Overlay selects one fixed overlay;
// Overrides contains only values explicitly supplied by an application CLI.
// Imports enables traversal of declarations in a selected main document; an
// absent source has no main declarations to traverse. Overlay selects one fixed
// overlay and requires filesystem context for NoSource. Overrides contain only
// values explicitly supplied by an application CLI.
type Options[T any] struct {
Definition Definition[T]
Imports bool
@@ -35,10 +37,13 @@ type Options[T any] struct {
placementMode placementMode
sequencePolicy sequencePolicy
deferAssembly bool
directiveContext rootDirectiveContext
importDefaultsValidated bool
}
// FilesystemContext supplies filesystem roots for imports and overlays.
// FilesystemContext supplies filesystem roots for imports and overlays. For an
// absent host input, Root is required and BaseDir may be omitted to use Root as
// the import base. Native contexts still require both BaseDir and Root.
type FilesystemContext struct {
BaseDir string
Root string
@@ -92,6 +97,10 @@ type Inspection struct {
baseline *document.Node
input *document.Node
inputOrigins map[string]Origin
structuralInput *document.Node
preCLI *document.Node
preCLIOrigins map[string]Origin
postDecodeApps []historyApplication
final *document.Node
immutableLookup *documentLookup
resolution []resolutionSnapshot
@@ -145,13 +154,13 @@ func Load[T any](source Source, secrets SecretBinding, supplied ...Options[T]) (
func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup secretLookup, supplied ...Options[T]) (*Loaded[T], error) {
options, diagnostics := selectOptions(source, supplied)
policy, policyDiagnostics := compileSequencePolicy(options.Definition.sequenceRegistrations)
policy, policyDiagnostics := compileSequenceIdentityPolicy(options.Definition.sequenceIdentityRegistrations)
diagnostics = append(diagnostics, policyDiagnostics...)
if len(diagnostics) != 0 {
return nil, (&diagnosticAccumulator{diagnostics: diagnostics}).loadError()
}
definition := options.Definition
definition.sequenceRegistrations = nil
definition.sequenceIdentityRegistrations = nil
request := loadRequest[T]{
main: loadInput{source: source, imports: options.Imports, filesystem: options.Filesystem, destination: options.destination.clone(), placementMode: options.placementMode},
definition: definition, importDefaults: snapshotImportDefaults(options.importDefaults), sequencePolicy: policy, overlay: options.Overlay,
@@ -168,11 +177,14 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode}
mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults)
mainOptions.sequencePolicy = request.sequencePolicy
mainOptions.deferAssembly = true
mainOptions.SourceVersionMode = request.sourceVersionMode
mainOptions.Definition.Version = request.protectionVersion()
mainOptions.Overrides = request.overrides
if request.ordinary {
mainOptions.Overlay = request.overlay
mainOptions.directiveContext = rootDirectiveContextSource
} else {
mainOptions.directiveContext = rootDirectiveContextLayered
}
var schema *schemaDescriptor
@@ -190,6 +202,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
accumulator.versionState = versionState
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
request.main.source = input.source
accumulator.sources = input.sources
accumulator.add(versionState.diagnostics...)
reportedVersionDiagnostics := len(versionState.diagnostics)
@@ -205,14 +218,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
return nil, accumulator.loadError()
}
}
if request.sourceVersionMode == sourceVersionModeDefault {
versionDiagnostics := checkVersion(request.definition.Version, input.document)
accumulator.add(versionDiagnostics...)
if hasErrors(versionDiagnostics) {
return nil, accumulator.loadError()
}
}
if request.ordinary && input.document != nil {
if request.ordinary && (input.document != nil || input.composition != nil) {
input.composition, diagnostics = compositionForInputWithPolicy(input, request.sequencePolicy)
if len(diagnostics) != 0 {
accumulator.add(diagnostics...)
@@ -220,7 +226,16 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
}
input.document = input.composition.root
}
if request.sourceVersionMode == sourceVersionModeDefault && request.ordinary {
versionDiagnostics := checkVersion(request.definition.Version, input.document)
accumulator.add(versionDiagnostics...)
if hasErrors(versionDiagnostics) {
return nil, accumulator.loadError()
}
}
if request.ordinary {
versionState.locateMainVersion(input.document)
mainOptions.Definition.Version = versionState.effectiveRule
input, diagnostics = applyInputLayers(request.main.source, mainOptions, input)
} else {
input, diagnostics = assembleLayeredInput(request, input, accounting)
@@ -228,9 +243,10 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
accumulator.add(versionState.diagnostics[reportedVersionDiagnostics:]...)
accumulator.sources = input.sources
accumulator.add(diagnostics...)
if len(diagnostics) != 0 {
if hasErrors(diagnostics) {
return nil, accumulator.loadError()
}
request.effectiveVersion = versionState.effectiveRule
if request.ordinary && input.parsed.source != nil {
baseSources := make([]SourceInfo, 0, len(input.sources))
for _, source := range input.sources {
@@ -253,20 +269,38 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
accumulator.add(problems...)
return nil, accumulator.loadError()
}
expanded, problems := expandInheritanceWithOrigins(composition.root, composition.origins, defaultProjectionLimits())
expanded, problems := expandInheritanceWithCollectionRules(
composition.root, composition.origins, composition.collectionPolicies, request.sequencePolicy, defaultProjectionLimits(),
)
if len(problems) != 0 {
accumulator.add(problems...)
return nil, accumulator.loadError()
}
if problem := inheritanceVersionDiagnostic(request.protectionVersion(), expanded); problem != nil {
effectiveVersion, problem := projectInheritanceVersion(request.protectionVersion(), expanded)
if problem != nil {
accumulator.add(*problem)
return nil, accumulator.loadError()
}
request.effectiveVersion = effectiveVersion
versionState.effectiveRule = effectiveVersion
input.composition = composition
input.inheritance = expanded
input.document = expanded.root
}
base := overridePreparationInput{root: input.document}
if input.composition != nil {
base.inputBytes = input.composition.inputBytes
base.inputNodes = input.composition.inputNodes
}
preparedOverrides, diagnostics := preparePostStructuralOverrides(base, input.sources, request.overrides, request.protectionVersion(), defaultImportLimits())
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()
}
input.sources = preparedOverrides.sources
accumulator.sources = input.sources
decodeDefinition := request.definition
decodeDefinition.Version = request.protectionVersion()
decoded, diagnostics := decodeLoadInput(schema, decodeDefinition, input)
@@ -274,12 +308,25 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
if len(diagnostics) != 0 {
return nil, accumulator.loadError()
}
resolved, diagnostics := resolveLoadInput(schema, request.definition, input, decoded)
decodeContext := decodeContext{}
if decodeDefinition.Version != nil {
decodeContext.versionPath = &decodeDefinition.Version.Path
}
assigned, issues := applyDecodedOverrides(schema, decoded, preparedOverrides, decodeContext)
if len(issues) != 0 {
for _, issue := range issues {
accumulator.add(diagnosticFromDecodeIssue(issue))
}
return nil, accumulator.loadError()
}
postDecode := preparePostDecodeOverrideContext(input, decoded, assigned)
resolved, diagnostics := resolveLoadInput(schema, request.definition, input, postDecode)
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()
}
if !historyNodesWithinLimit(decoded.baseline, decoded.final, resolved.final, resolved.applications, resolved.snapshots, input.inheritance) {
historyApplications := append(cloneHistoryApplications(resolved.applications), cloneHistoryApplications(resolved.postDecodeApps)...)
if !historyNodesWithinLimit(decoded.baseline, decoded.final, resolved.final, historyApplications, resolved.snapshots, input.inheritance) {
accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"})
return nil, accumulator.loadError()
}
@@ -290,13 +337,18 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
accumulator.add(*ownershipIssue)
return nil, accumulator.loadError()
}
secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, input.document, lookup)
secretResolution := resolveSecretReferences(reflect.ValueOf(&configuration).Elem(), schema, resolved.referenceInput, lookup)
accumulator.add(secretResolution.diagnostics...)
if hasErrors(secretResolution.diagnostics) {
return nil, accumulator.loadError()
}
identityDiagnostics := validateFinalSequenceIdentities(resolved.referenceInput, request.sequencePolicy)
accumulator.add(identityDiagnostics...)
if hasErrors(identityDiagnostics) {
return nil, accumulator.loadError()
}
if request.definition.Validate != nil {
diagnostics = runValidation(&configuration, request.definition.Validate, schema, resolved.final, input.document, secretResolution.lookups)
diagnostics = runValidation(&configuration, request.definition.Validate, schema, resolved.final, resolved.referenceInput, secretResolution.lookups)
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()

View File

@@ -8,6 +8,7 @@ import (
// assembledInput is the source/document handoff across schema and version checks.
type assembledInput struct {
source Source
parsed parsedSource
document *document.Node
composition *composedDocument
@@ -16,36 +17,42 @@ type assembledInput struct {
sources []SourceInfo
contexts map[string]ResourceContext
versionState *sourceVersionState
accounting *importAccounting
}
// acquiredContribution keeps a source-local payload beside its placed document.
// The latter is used for composition; the former preserves the acquisition
// boundary for later scoped assembly and evidence.
type acquiredContribution struct {
source SourceInfo
identity importNodeKey
role importContributionRole
scopeOwner importNodeKey
payload *document.Node
document *document.Node
destination Path
relativeDestination Path
processing *sourceVersionProcessingResult
nested *composedDocument
pendingItem *acquisitionScope
placementCharged bool
source SourceInfo
identity importNodeKey
role importContributionRole
scopeOwner importNodeKey
payload *document.Node
document *document.Node
destination Path
relativeDestination Path
processing *sourceVersionProcessingResult
sequenceDeclarations []sequenceDeclarationSpec
nested *composedDocument
pendingItem *acquisitionScope
placementCharged bool
}
// acquisitionScope is the ordered strict mapping scope produced by one input
// and its ordinary imports.
type acquisitionScope struct {
owner importNodeKey
destination Path
contributions []acquiredContribution
owner importNodeKey
destination Path
contributions []acquiredContribution
collectionPolicies collectionPolicyScope
}
func cloneAcquisitionScope(scope acquisitionScope) acquisitionScope {
cloned := acquisitionScope{owner: scope.owner, destination: scope.destination.clone()}
cloned := acquisitionScope{
owner: scope.owner, destination: scope.destination.clone(),
collectionPolicies: cloneCollectionPolicyScope(scope.collectionPolicies),
}
if scope.contributions != nil {
cloned.contributions = make([]acquiredContribution, len(scope.contributions))
for index, contribution := range scope.contributions {
@@ -54,7 +61,8 @@ func cloneAcquisitionScope(scope acquisitionScope) acquisitionScope {
role: contribution.role, scopeOwner: contribution.scopeOwner, payload: contribution.payload,
document: contribution.document, destination: contribution.destination.clone(),
relativeDestination: contribution.relativeDestination.clone(), processing: contribution.processing,
nested: cloneComposedDocument(contribution.nested), pendingItem: cloneAcquisitionScopePointer(contribution.pendingItem),
sequenceDeclarations: cloneSequenceDeclarations(contribution.sequenceDeclarations),
nested: cloneComposedDocument(contribution.nested), pendingItem: cloneAcquisitionScopePointer(contribution.pendingItem),
placementCharged: contribution.placementCharged,
}
}
@@ -94,20 +102,41 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
}
func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, role sourceVersionRole) (assembledInput, []Diagnostic) {
sourceID := "source-" + strconv.Itoa(accounting.sources+1)
selectedSource := source
if source.kind == sourceSearch {
resolved, failedSource, diagnostics := resolveSearchInput(source, sourceID, defaultDiscoveryDirectoryHelpers(), defaultDiscoveryHostProbe())
if len(diagnostics) != 0 {
if failedSource != nil {
failedSource.Destination = destination.clone()
failedSource.placementMode = options.placementMode
}
return assembledInput{source: resolved, sources: sourceInfos(failedSource), versionState: versionState, accounting: accounting}, diagnostics
}
selectedSource = resolved
if options.Filesystem != nil {
if diagnostics := filesystemOptionDiagnostics(selectedSource, options.Filesystem); len(diagnostics) != 0 {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, diagnostics
}
}
if selectedSource.kind == sourceNone && activeOverlay(options.Overlay) && options.Filesystem == nil {
return assembledInput{source: selectedSource, versionState: versionState, accounting: accounting}, []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context")}
}
}
limits := defaultImportLimits()
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
parsed, diagnostics := parseSource(source, "source-"+strconv.Itoa(accounting.sources+1), parseLimits)
parsed, diagnostics := parseSource(selectedSource, sourceID, parseLimits)
if parsed.source != nil {
parsed.source.Destination = destination.clone()
parsed.source.placementMode = options.placementMode
}
input := assembledInput{parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState}
input := assembledInput{source: selectedSource, parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState, accounting: accounting}
if len(diagnostics) != 0 {
input.sources = sourceInfos(parsed.source)
// Mapping value issues already identify application content. Other early
// failures describe source syntax or access, not a placed value.
if source.kind == sourceMapping && source.mapIssue != nil && len(destination.segments) != 0 {
if selectedSource.kind == sourceMapping && selectedSource.mapIssue != nil && len(destination.segments) != 0 {
for index := range diagnostics {
relative := diagnostics[index].Path
segments := clonePathSegments(destination.segments, len(relative.segments))
@@ -116,13 +145,70 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
}
return input, diagnostics
}
directiveContext := options.directiveContext
if directiveContext == 0 {
directiveContext = rootDirectiveContextImportsOnly
}
if !options.Imports && parsed.document != nil {
if diagnostics := disabledDirectiveDiagnostics(parsed.document, Path{}); len(diagnostics) != 0 {
return input, diagnostics
var directiveDiagnostics []Diagnostic
if directiveContext == rootDirectiveContextSource || directiveContext == rootDirectiveContextLayered {
directiveDiagnostics = disabledRootImportDiagnostics(parsed.document)
} else {
directiveDiagnostics = disabledDirectiveDiagnostics(parsed.document, Path{})
}
if len(directiveDiagnostics) != 0 {
return input, directiveDiagnostics
}
}
var versionProcessing *sourceVersionProcessingResult
if versionState != nil && versionState.mode != sourceVersionModeDefault {
if directiveContext == rootDirectiveContextSource {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
var processed sourceVersionProcessingResult
var failures []Diagnostic
if versionState != nil {
processed, failures = versionState.processWithDirectiveContext(parsed.document, role, destination, sourceID, options.placementMode, directiveContext)
} else {
processed = processSourceVersionWithDirectiveContext(parsed.document, options.Definition.Version, role, destination, sourceVersionModeDefault, directiveContext)
failures = processed.diagnostics
}
if len(failures) != 0 {
return input, failures
}
versionProcessing = &processed
if role == sourceVersionMain && versionState != nil {
versionState.mainProcessing = &processed
}
} else if directiveContext == rootDirectiveContextLayered {
extracted, failures := extractRootDirectives(parsed.document, rootDirectiveContextLayered)
if len(failures) != 0 {
return input, failures
}
processed := sourceVersionProcessingResult{
payload: extracted.body, imports: extracted.imports, sequences: extracted.sequences,
directiveContext: rootDirectiveContextLayered,
}
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
versionProcessed, versionFailures := versionState.process(extracted.body, role, destination, sourceID, options.placementMode)
if len(versionFailures) != 0 {
return input, versionFailures
}
versionProcessed.imports = extracted.imports
versionProcessed.sequences = append(cloneSequenceDeclarations(extracted.sequences), versionProcessed.sequences...)
versionProcessed.directiveContext = rootDirectiveContextLayered
processed = versionProcessed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
versionProcessing = &processed
} else if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
@@ -136,11 +222,11 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
versionState.mainProcessing = &processed
}
}
if options.Imports {
if options.Imports && selectedSource.kind != sourceNone {
if options.deferAssembly {
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessingDeferred(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy, true)
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessingDeferred(selectedSource, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy, true)
} else {
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy)
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(selectedSource, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing, options.sequencePolicy)
}
if len(diagnostics) != 0 {
return input, diagnostics
@@ -154,8 +240,8 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
if versionProcessing != nil {
input.document = versionProcessing.payload
}
if options.Filesystem != nil && !activeOverlay(options.Overlay) {
if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil {
if options.Filesystem != nil && (selectedSource.kind == sourceNone || !activeOverlay(options.Overlay)) {
if diagnostic := validateFilesystemContext(selectedSource, options.Filesystem, parsed); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
}
@@ -168,6 +254,10 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
localPayload := input.document
input.document = placed
identity := importKeyForMainSource(parsed.source)
var sequenceDeclarations []sequenceDeclarationSpec
if versionProcessing != nil {
sequenceDeclarations = cloneSequenceDeclarations(versionProcessing.sequences)
}
input.acquisition = []acquisitionScope{{
owner: identity,
destination: destination.clone(),
@@ -175,6 +265,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
source: cloneSourceInfo(*parsed.source), payload: localPayload,
identity: identity, role: importRoleMain, scopeOwner: identity,
document: placed, destination: destination.clone(), processing: versionProcessing,
sequenceDeclarations: sequenceDeclarations,
}},
}}
rawNodes := countInputNodes(parsed.document)
@@ -182,7 +273,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+rawNodes+placementNodes > limits.nodes {
return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the aggregate input limit")}
}
if diagnostic := registerAcquiredSource(source, options.Filesystem, parsed.source, visited, hostToken); diagnostic != nil {
if diagnostic := registerAcquiredSource(selectedSource, options.Filesystem, parsed.source, visited, hostToken); diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
accounting.sources++
@@ -190,7 +281,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
accounting.nodes += rawNodes + placementNodes
}
if !activeOverlay(options.Overlay) || options.Imports {
contexts, diagnostic := contextsForAcquiredSources(source, options.Filesystem, parsed.source, input.sources)
contexts, diagnostic := contextsForAcquiredSources(selectedSource, options.Filesystem, parsed.source, input.sources)
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
}
@@ -218,7 +309,7 @@ func registerAcquiredSource(source Source, context *FilesystemContext, info *Sou
}
func applyInputLayers[T any](source Source, options Options[T], input assembledInput) (assembledInput, []Diagnostic) {
if !activeOverlay(options.Overlay) && len(options.Overrides) == 0 {
if !activeOverlay(options.Overlay) {
return input, nil
}
composition, diagnostics := compositionForInputWithPolicy(input, options.sequencePolicy)
@@ -229,7 +320,7 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI
if activeOverlay(options.Overlay) {
input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(
source, options.Filesystem, input.parsed, assembly, input.sources,
options.Overlay, options.Definition.Version, defaultImportLimits(), input.versionState,
options.Overlay, options.Definition.Version, defaultImportLimits(), input.versionState, options.sequencePolicy,
)
if len(diagnostics) != 0 {
return input, diagnostics
@@ -240,33 +331,44 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI
}
mergeResourceContexts(&input.contexts, contexts)
}
if len(options.Overrides) != 0 {
input.sources, diagnostics = assembly.applyOverrides(
input.sources, options.Overrides, options.Definition.Version, defaultImportLimits(),
)
if len(diagnostics) != 0 {
return input, diagnostics
}
}
input.composition = assembly.current
input.document = assembly.current.root
return input, nil
}
// compositionForInput initializes one handoff for overlays and CLI, including
// defaults-only CLI input. Raw source bytes and mapping keys retain their budgets.
// compositionForInput initializes the structural handoff used by inheritance
// expansion. Raw source bytes and mapping keys retain their budgets.
func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic) {
return compositionForInputWithPolicy(input, sequencePolicy{})
}
func compositionForInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) {
return compositionForInputWithPolicies(input, collectionPolicyScope{}, policy)
}
func compositionForInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
if input.composition != nil {
if input.composition.root == nil && len(input.composition.acquisition) != 0 {
return composeAcquiredInputWithPolicies(input, lower, policy)
}
return input.composition, nil
}
if len(input.acquisition) != 0 {
acquired := &composedDocument{acquisition: cloneAcquisitionScopes(input.acquisition)}
if input.accounting != nil {
acquired.inputBytes = input.accounting.bytes
acquired.inputNodes = input.accounting.nodes
} else {
acquired.inputBytes = input.parsed.bytes
acquired.inputNodes = countInputNodes(input.parsed.document)
}
input.composition = acquired
return composeAcquiredInputWithPolicies(input, lower, policy)
}
if input.document == nil {
return &composedDocument{root: &document.Node{Kind: document.Mapping}, origins: make(map[string]Origin)}, nil
}
composition, diagnostics := composeDocumentsWithPolicy(policy, input.document)
composition, diagnostics := composeDocumentsWithCollectionPolicy(policy, lower, nil, input.document)
if len(diagnostics) != 0 {
return nil, diagnostics
}

View File

@@ -37,15 +37,25 @@ type loadInput struct {
type loadGroup struct{ inputs []loadInput }
type acquiredUnderlayGroup struct {
index int
scopes []acquisitionScope
importGraph importInclusionGraph
sources []SourceInfo
contexts map[string]ResourceContext
inputBytes int64
inputNodes int
}
func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
request.ordinary = false
var diagnostics []Diagnostic
request.importDefaults = snapshotImportDefaults(request.importDefaults)
diagnostics = append(diagnostics, validateImportDefaults(request.importDefaults)...)
policy, policyDiagnostics := compileSequencePolicy(request.definition.sequenceRegistrations)
policy, policyDiagnostics := compileSequenceIdentityPolicy(request.definition.sequenceIdentityRegistrations)
request.sequencePolicy = policy
diagnostics = append(diagnostics, policyDiagnostics...)
request.definition.sequenceRegistrations = nil
request.definition.sequenceIdentityRegistrations = nil
mainOptions, mainDiagnostics := selectOptions(request.main.source, []Options[T]{{
Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(),
SourceVersionMode: request.sourceVersionMode, placementMode: request.main.placementMode,
@@ -82,46 +92,114 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
}
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
groups := make([]*composedDocument, 0, len(request.underlays))
var assembly *assemblyAccumulator
mainSourceCount := len(input.sources)
mainEvidence := input.versionState.evidenceSince(0)
projectVersion := sequenceVersionPath(request.protectionVersion())
for index, group := range request.underlays {
composition, sources, contexts, diagnostics := acquireUnderlayGroupWithPolicyVersion(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
input.sources = append(input.sources, sources...)
mergeResourceContexts(&input.contexts, contexts)
evidenceStart := len(input.versionState.evidenceSince(0))
acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
mergeResourceContexts(&input.contexts, acquired.contexts)
if len(diagnostics) != 0 {
input.sources = append(input.sources, acquired.sources...)
return input, diagnostics
}
var lower collectionPolicyScope
if assembly != nil && assembly.current != nil {
lower = assembly.current.collectionPolicies
}
protectionRule := request.protectionVersion()
if projectVersion {
// The main item's final position is known only after tier assembly.
// Source-local checks still run during acquisition above.
protectionRule = nil
}
composition, diagnostics := composeUnderlayGroup(acquired, protectionRule, request.sequencePolicy, lower)
sourceStart := len(input.sources)
input.sources = append(input.sources, acquired.sources...)
if len(diagnostics) != 0 {
return input, diagnostics
}
if composition != nil {
groups = append(groups, composition)
if assembly == nil {
assembly = newAssemblyAccumulator(composition)
} else {
var declarations []sequenceDeclarationSpec
for _, scope := range composition.acquisition {
declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
}
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
composition.root, composition.origins, composition.collectionPolicies, composition.applications, LayerUnderlay,
request.sequencePolicy, declarations,
)
if len(diagnostics) != 0 {
return input, diagnostics
}
projectTierSourceDestinations(input.sources[sourceStart:], input.versionState.evidenceSince(evidenceStart), operations)
}
}
}
var assembly *assemblyAccumulator
for _, group := range groups {
if assembly == nil {
assembly = newAssemblyAccumulator(group)
continue
mainPresent := input.document != nil || input.composition != nil || len(input.acquisition) != 0
var mainComposition *composedDocument
if mainPresent {
var lower collectionPolicyScope
if assembly != nil && assembly.current != nil {
lower = assembly.current.collectionPolicies
}
if diagnostics := assembly.applyOverlay(group.root, group.origins, group.applications, LayerUnderlay); len(diagnostics) != 0 {
return input, diagnostics
}
}
if input.document != nil {
mainComposition, diagnostics := compositionForInputWithPolicy(input, request.sequencePolicy)
var diagnostics []Diagnostic
mainComposition, diagnostics = compositionForInputWithPolicies(input, lower, request.sequencePolicy)
if len(diagnostics) != 0 {
return input, diagnostics
}
if assembly == nil {
assembly = newAssemblyAccumulator(mainComposition)
} else if diagnostics := assembly.applyOverlay(mainComposition.root, mainComposition.origins, mainComposition.applications, LayerBase); len(diagnostics) != 0 {
return input, diagnostics
}
var versionDiagnostics []Diagnostic
if request.sourceVersionMode == sourceVersionModeDefault {
var mainRoot *document.Node
if mainComposition != nil {
mainRoot = mainComposition.root
} else if input.document != nil {
mainRoot = input.document
}
versionDiagnostics = checkVersion(request.definition.Version, mainRoot)
if hasErrors(versionDiagnostics) {
return input, versionDiagnostics
}
}
if assembly == nil && (activeOverlay(request.overlay) || len(request.overrides) != 0) {
if mainComposition != nil {
input.versionState.locateMainVersion(mainComposition.root)
if assembly == nil {
assembly = newAssemblyAccumulator(mainComposition)
} else {
var declarations []sequenceDeclarationSpec
for _, scope := range mainComposition.acquisition {
declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
}
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
mainComposition.root, mainComposition.origins, mainComposition.collectionPolicies, mainComposition.applications, LayerBase,
request.sequencePolicy, declarations,
)
if len(diagnostics) != 0 {
return input, append(versionDiagnostics, diagnostics...)
}
projectTierSourceDestinations(input.sources[:mainSourceCount], mainEvidence, operations)
input.versionState.projectMainVersion(operations)
}
}
if input.versionState != nil {
request.effectiveVersion = input.versionState.effectiveRule
}
if projectVersion && assembly != nil {
if diagnostic := projectedUnderlayVersionDiagnostic(assembly.current.applications, input.sources, request.protectionVersion()); diagnostic != nil {
return input, append(versionDiagnostics, *diagnostic)
}
}
if assembly == nil && activeOverlay(request.overlay) {
composition, _ := compositionForInput(assembledInput{})
assembly = newAssemblyAccumulator(composition)
}
if assembly == nil {
return input, nil
return input, versionDiagnostics
}
assembly.current.inputBytes = accounting.bytes
assembly.current.inputNodes = accounting.nodes
@@ -131,37 +209,23 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
selection := Source{kind: sourceBytes}
selected := parsedSource{document: &document.Node{Kind: document.Mapping}, source: &SourceInfo{}}
var diagnostics []Diagnostic
input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState)
overlayPolicy := []sequencePolicy{request.sequencePolicy}
input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState, overlayPolicy...)
if len(diagnostics) != 0 {
return input, diagnostics
return input, append(versionDiagnostics, diagnostics...)
}
contexts, diagnostic := contextsForAcquiredSources(selection, request.overlayFilesystem, selected.source, input.sources[len(input.sources)-1:])
if diagnostic != nil {
return input, []Diagnostic{*diagnostic}
return input, append(versionDiagnostics, *diagnostic)
}
mergeResourceContexts(&input.contexts, contexts)
}
if len(request.overrides) != 0 {
var diagnostics []Diagnostic
input.sources, diagnostics = assembly.applyOverrides(input.sources, request.overrides, request.protectionVersion(), defaultImportLimits())
if len(diagnostics) != 0 {
return input, diagnostics
}
}
input.composition = assembly.current
input.document = assembly.current.root
return input, nil
return input, versionDiagnostics
}
func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
return acquireUnderlayGroupWithVersion(group, index, rule, accounting, nil)
}
func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
return acquireUnderlayGroupWithPolicyVersion(group, index, rule, accounting, versionState, sequencePolicy{})
}
func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) {
initialBytes, initialNodes := accounting.bytes, accounting.nodes
visited := make(map[resourceIdentity]importRoute)
hostToken := &namespaceToken{}
@@ -173,15 +237,14 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
}
}
}
var scopes []acquisitionScope
acquiredGroup := &acquiredUnderlayGroup{index: index}
inclusionGraph := newImportInclusionGraph()
var sources []SourceInfo
var contexts map[string]ResourceContext
for inputIndex, contribution := range group.inputs {
options := Options[struct{}]{
Imports: contribution.imports, Filesystem: contribution.filesystem,
Definition: Definition[struct{}]{Version: rule}, destination: contribution.destination.clone(),
placementMode: contribution.placementMode, sequencePolicy: policy, deferAssembly: true,
directiveContext: rootDirectiveContextLayered,
importDefaults: func() importDefaults {
if versionState == nil {
return importDefaults{}
@@ -193,11 +256,11 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for _, source := range acquired.sources {
source.Layer = LayerUnderlay
source.UnderlayIndex = index
sources = append(sources, source)
acquiredGroup.sources = append(acquiredGroup.sources, source)
}
mergeResourceContexts(&contexts, acquired.contexts)
mergeResourceContexts(&acquiredGroup.contexts, acquired.contexts)
if len(diagnostics) != 0 {
return nil, sources, contexts, projectPriorInputDiagnosticPaths(diagnostics, scopes)
return acquiredGroup, projectPriorInputDiagnosticPaths(diagnostics, acquiredGroup.scopes)
}
if acquired.document == nil && len(acquired.acquisition) == 0 {
continue
@@ -219,59 +282,66 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for scopeIndex := range inputScopes {
reanchorAcquisitionScope(&inputScopes[scopeIndex], keyMap, anchors)
}
scopes = append(scopes, inputScopes...)
acquiredGroup.scopes = append(acquiredGroup.scopes, inputScopes...)
mergeImportInclusionGraph(&inclusionGraph, inputGraph)
}
groupScope := scheduleAcquisitionGroup(scopes, inclusionGraph)
traversal := importTraversal{accounting: accounting, limits: defaultImportLimits(), rule: rule, importGraph: inclusionGraph}
if diagnostics := traversal.composePendingItems(&groupScope); len(diagnostics) != 0 {
return nil, sources, contexts, diagnostics
acquiredGroup.importGraph = inclusionGraph.clone()
acquiredGroup.inputBytes = accounting.bytes - initialBytes
acquiredGroup.inputNodes = accounting.nodes - initialNodes
return acquiredGroup, nil
}
func composeUnderlayGroup(acquired *acquiredUnderlayGroup, rule *VersionRule, policy sequencePolicy, lower collectionPolicyScope) (*composedDocument, []Diagnostic) {
if acquired == nil {
return nil, nil
}
groupScope := scheduleAcquisitionGroup(acquired.scopes, acquired.importGraph)
if len(groupScope.contributions) == 0 {
return nil, nil
}
groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(acquired.index, 0), resourceKind: "scope"}
for contributionIndex := range groupScope.contributions {
groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
}
accounting := &importAccounting{
sources: len(acquired.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes,
}
composition, diagnostics := composeAcquiredScopeWithPolicy(
&groupScope, lower, policy, acquired.importGraph, acquired.sources, nil, nil, accounting,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
var documents []*document.Node
for _, contribution := range groupScope.contributions {
documents = append(documents, contribution.document)
}
if len(documents) == 0 {
return nil, sources, contexts, nil
}
composition, diagnostics := composeDocumentsWithPolicy(policy, documents...)
if len(diagnostics) != 0 {
return nil, sources, contexts, diagnostics
}
if composition != nil && rule != nil {
ranges := indexSequenceRanges(composition.sequenceRanges)
for sourceIndex, document := range documents {
for sourceIndex, input := range documents {
if groupScope.contributions[sourceIndex].role == importRoleMain {
continue
}
if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
location := publicLocation(node.Location)
return nil, sources, contexts, []Diagnostic{{
return nil, []Diagnostic{{
Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &location,
}}
}
}
for sourceIndex, document := range documents {
if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
for sourceIndex, input := range documents {
if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
location := publicLocation(node.Location)
return nil, sources, contexts, []Diagnostic{{
return nil, []Diagnostic{{
Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location,
}}
}
}
}
composition.inputBytes = accounting.bytes - initialBytes
composition.inputNodes = accounting.nodes - initialNodes
groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(index, 0), resourceKind: "scope"}
for contributionIndex := range groupScope.contributions {
groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
}
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(groupScope)}
composition.importGraph = inclusionGraph.clone()
projectNestedEvidence(composition, groupScope)
projectAcquiredDestinations(composition, groupScope, sources)
composition.inputBytes = acquired.inputBytes
composition.inputNodes = acquired.inputNodes
for pointer, origin := range composition.origins {
origin.Layer = LayerUnderlay
composition.origins[pointer] = origin
@@ -279,7 +349,7 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for index := range composition.applications {
composition.applications[index].layer = LayerUnderlay
}
return composition, sources, contexts, nil
return composition, nil
}
func nativeTokenForLoadInput(input loadInput) *namespaceToken {

View File

@@ -46,7 +46,7 @@ items: [{id: b}]
{source: File(zRoot), imports: true},
{source: File(bRoot), imports: true},
}}},
definition: Definition[map[string]any]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}},
definition: Definition[map[string]any]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}},
}
result, err := testGroupedLoad(request)
if err != nil {
@@ -73,6 +73,50 @@ items: [{id: b}]
}
}
func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) {
filesystem := NewFilesystem("handoff", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")},
"child.yml": &fstest.MapFile{Data: []byte("child: retained\n")},
})
accounting := &importAccounting{}
group := loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}
acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, accounting, nil, sequencePolicy{})
if len(diagnostics) != 0 {
t.Fatalf("acquire underlay group: %#v", diagnostics)
}
if acquired == nil || acquired.index != 2 || len(acquired.scopes) != 1 || len(acquired.scopes[0].contributions) != 2 {
t.Fatalf("acquisition handoff lost its source scope: %#v", acquired)
}
if len(acquired.sources) != 2 || acquired.sources[0].UnderlayIndex != 2 || acquired.sources[1].ID != "source-2" {
t.Fatalf("acquired source evidence = %#v", acquired.sources)
}
if acquired.inputBytes != int64(len("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")+len("child: retained\n")) {
t.Fatalf("group input byte accounting = %d", acquired.inputBytes)
}
composition, diagnostics := composeUnderlayGroup(acquired, nil, sequencePolicy{}, collectionPolicyScope{})
if len(diagnostics) != 0 {
t.Fatalf("compose acquired group: %#v", diagnostics)
}
for key, want := range map[string]string{"main": "retained", "child": "retained"} {
value, present := documentAtPath(composition.root, Keys(key))
if !present || value.Text != want {
t.Errorf("composed %s = %#v, %v; want %q", key, value, present, want)
}
}
if composition.inputBytes != acquired.inputBytes || len(composition.acquisition) != 1 || len(composition.acquisition[0].contributions) != 2 {
t.Fatalf("composed group handoff = %#v", composition)
}
empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, accounting, nil, sequencePolicy{})
if len(diagnostics) != 0 || len(empty.scopes) != 0 {
t.Fatalf("empty group acquisition = %#v, %#v", empty, diagnostics)
}
if composition, diagnostics := composeUnderlayGroup(empty, nil, sequencePolicy{}, collectionPolicyScope{}); len(diagnostics) != 0 || composition != nil {
t.Fatalf("empty group composition = %#v, %#v", composition, diagnostics)
}
}
func TestUnderlayNamespaceAnchorsUseInputPositionsAndFilesystemIdentity(t *testing.T) {
firstNamespace := NewFilesystem("same-label", fstest.MapFS{
"root.yml": &fstest.MapFile{Data: []byte("items: [{id: first}]\n")},
@@ -88,7 +132,7 @@ func TestUnderlayNamespaceAnchorsUseInputPositionsAndFilesystemIdentity(t *testi
{source: secondNamespace.File("root.yml")},
{source: firstNamespace.File("later.yml")},
}}},
definition: Definition[map[string]any]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}},
definition: Definition[map[string]any]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}},
}
result, err := testGroupedLoad(request)
if err != nil {
@@ -119,7 +163,7 @@ func TestUnderlayItemConflictUsesScheduledGroupIndex(t *testing.T) {
{source: File(zItem), destination: Keys("items"), placementMode: placementModeSequenceItem},
{source: File(aItem), destination: Keys("items"), placementMode: placementModeSequenceItem},
}}},
definition: Definition[map[string]any]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}},
definition: Definition[map[string]any]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}},
}
_, err := testGroupedLoad(request)
loadError, ok := err.(*LoadError)
@@ -163,7 +207,7 @@ func TestUnderlayItemHelperConflictUsesScheduledGroupIndex(t *testing.T) {
{source: File(zBad), imports: true, destination: Keys("items"), placementMode: placementModeSequenceItem},
{source: File(aGood), destination: Keys("items"), placementMode: placementModeSequenceItem},
}}},
definition: Definition[map[string]any]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}},
definition: Definition[map[string]any]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}},
}
_, err := testGroupedLoad(request)
loadError, ok := err.(*LoadError)
@@ -311,6 +355,69 @@ func TestGroupedLoadAbsentMainAndVersionProtection(t *testing.T) {
}
}
func TestLayeredCollectionPolicyFlowsBetweenTiers(t *testing.T) {
t.Parallel()
policyOnly := Bytes("lower-policy", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\n"))
duplicate := func(name string) loadInput {
return loadInput{source: Bytes(name, []byte("items: [{id: repeated}]\n"))}
}
request := loadRequest[map[string]any]{
main: loadInput{source: NoSource()},
underlays: []loadGroup{
{inputs: []loadInput{{source: policyOnly}}},
{inputs: []loadInput{duplicate("first-peer"), duplicate("second-peer")}},
},
}
if _, err := testGroupedLoad(request); err != nil && groupedErrorCode(t, err) != CodeComposeConflict {
loadErr, _ := err.(*LoadError)
t.Fatalf("higher strict group code = %s, want %s: %#v", groupedErrorCode(t, err), CodeComposeConflict, loadErr.Diagnostics())
} else if err == nil {
t.Fatal("higher strict group accepted duplicate inherited identities")
}
request = loadRequest[map[string]any]{
main: loadInput{source: NoSource()},
underlays: []loadGroup{
{inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: repeated}]\n"))}}},
{inputs: []loadInput{duplicate("higher")}},
},
}
if _, err := testGroupedLoad(request); err == nil || groupedErrorCode(t, err) != CodeSequenceConflict {
t.Fatalf("higher tier did not inherit explicit append mode: %v", err)
}
request = loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte("configkit:\n sequences:\n - destination: [items]\n key: name\n mode: replace\nitems: [{name: higher}]\n"))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: lower}]\n"))}}}},
}
result, err := testGroupedLoad(request)
if err != nil || !reflect.DeepEqual(result.Config["items"], []any{map[string]any{"name": "higher"}}) {
var diagnostics []Diagnostic
if loadErr, ok := err.(*LoadError); ok {
diagnostics = loadErr.Diagnostics()
}
t.Fatalf("replacement did not retire lower identity: %#v, %v (%#v)", result, err, diagnostics)
}
filesystem := NewFilesystem("overlay", fstest.MapFS{
"overlays/testing.yml": &fstest.MapFile{Data: []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: overlay}]\n")},
})
definition := Definition[map[string]any]{sequenceIdentityRegistrations: []sequenceIdentityRegistration{{selector: sequenceIdentitySelector{literalSequenceSelector("items")}, identityFields: []string{"id"}}}}
request = loadRequest[map[string]any]{
main: loadInput{source: Bytes("main", []byte("items: [{id: main}]\n"))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("items: [{id: lower}]\n"))}}}},
definition: definition, sequencePolicy: mustCompileSequenceIdentityPolicy(t, definition.sequenceIdentityRegistrations),
overlay: "testing", overlayFilesystem: &FilesystemContext{Root: ".", BaseDir: ".", FS: &filesystem},
}
result, err = testGroupedLoad(request)
if err != nil || !reflect.DeepEqual(result.Config["items"], []any{
map[string]any{"id": "lower"}, map[string]any{"id": "main"}, map[string]any{"id": "overlay"},
}) {
t.Fatalf("layered selected overlay sequence policy = %#v, %v", result, err)
}
}
func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) {
main := loadInput{source: Bytes("main", []byte("port: 5\n"))}
request := loadRequest[groupedConfig]{
@@ -349,18 +456,27 @@ func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) {
accounting := &importAccounting{sources: maxImportSources - 1}
group := loadGroup{inputs: []loadInput{{source: Mapping("first", map[string]any{})}, {source: Mapping("second", map[string]any{})}}}
_, _, _, diagnostics := acquireUnderlayGroup(group, 1, nil, accounting)
_, _, _, diagnostics := testAcquireUnderlayGroup(group, 1, nil, accounting)
if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
t.Fatalf("shared source budget = %#v", diagnostics)
}
accounting = &importAccounting{bytes: maxTotalBytes - 2}
group.inputs = []loadInput{{source: Mapping("first", map[string]any{"port": "one"})}}
_, _, _, diagnostics = acquireUnderlayGroup(group, 1, nil, accounting)
_, _, _, diagnostics = testAcquireUnderlayGroup(group, 1, nil, accounting)
if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
t.Fatalf("shared byte budget = %#v", diagnostics)
}
}
func testAcquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, nil, sequencePolicy{})
if len(diagnostics) != 0 {
return nil, acquired.sources, acquired.contexts, diagnostics
}
composition, diagnostics := composeUnderlayGroup(acquired, rule, sequencePolicy{}, collectionPolicyScope{})
return composition, acquired.sources, acquired.contexts, diagnostics
}
func TestGroupedLoadInvalidIntermediateTypeCanBeReplaced(t *testing.T) {
request := loadRequest[groupedConfig]{
main: loadInput{source: Bytes("main", []byte("port: 9\n"))},
@@ -402,11 +518,11 @@ func TestGroupedAccountingChargesMixedOccurrencesOnce(t *testing.T) {
"child-two.yml": &fstest.MapFile{Data: []byte(child)},
})
accounting := &importAccounting{}
first, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("one.yml"), imports: true}}}, 1, nil, accounting)
first, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("one.yml"), imports: true}}}, 1, nil, accounting)
if len(diagnostics) != 0 || first == nil || len(sources) != 2 || accounting.sources != 2 || accounting.bytes != int64(len(one)+len(child)) {
t.Fatalf("first acquisition = %#v, %#v, %#v, accounting %#v", first, sources, diagnostics, accounting)
}
second, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("two.yml"), imports: true}}}, 2, nil, accounting)
second, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("two.yml"), imports: true}}}, 2, nil, accounting)
if len(diagnostics) != 0 || second == nil || len(sources) != 2 || sources[0].ID != "source-3" || sources[1].ID != "source-4" || accounting.sources != 4 || accounting.bytes != int64(len(one)+len(two)+2*len(child)) {
t.Fatalf("second acquisition = %#v, %#v, %#v, accounting %#v", second, sources, diagnostics, accounting)
}
@@ -433,7 +549,7 @@ func TestGroupedGlobEntryBudgetIsShared(t *testing.T) {
"parts/b.yml": &fstest.MapFile{Data: []byte("right: two\n")},
})
accounting := &importAccounting{entries: maxGlobEntries - 1}
_, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}, 1, nil, accounting)
_, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}, 1, nil, accounting)
if len(diagnostics) == 0 || diagnostics[0].Code != CodeImportLimitExceeded || accounting.entries <= maxGlobEntries {
t.Fatalf("shared glob budget = %#v, accounting %#v", diagnostics, accounting)
}

View File

@@ -15,6 +15,11 @@ type resolvedLoad[T any] struct {
final *document.Node
snapshots []resolutionSnapshot
applications []historyApplication
postDecodeApps []historyApplication
structuralInput *document.Node
preCLI *document.Node
preCLIOrigins map[string]Origin
referenceInput *document.Node
versionEvidence []*sourceVersionEvidence
inheritance *inheritanceScope
contexts map[string]ResourceContext
@@ -46,24 +51,16 @@ func decodeLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
return decoded, nil
}
func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput, decoded decodedConfiguration) (resolvedLoad[T], []Diagnostic) {
func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T], input assembledInput, prepared postDecodeOverrideContext) (resolvedLoad[T], []Diagnostic) {
var diagnostics []Diagnostic
decoded := prepared.decoded
configuration := decoded.value.Interface().(T)
metadata := metadataFromDecoded(decoded)
if input.inheritance != nil {
applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, input.composition.overlays, input.inheritance)
} else if input.composition != nil {
applyOverlayMetadata(&metadata, input.composition.overlays)
}
metadata := prepared.metadata
final := decoded.final
referenceInput := prepared.referenceInput
var resolutionSnapshots []resolutionSnapshot
if definition.Resolve != nil {
sources := input.sources
if input.versionState != nil {
sources = sourceInfosWithVersionEvidence(sources, input.versionState.evidence, nil, true)
}
entry := newResolutionInput(metadata, decoded.final, sources, input.contexts)
resolved := runResolution(&configuration, definition.Resolve, schema, decoded, input.document, entry)
resolved := runResolution(&configuration, definition.Resolve, schema, decoded, referenceInput, prepared.resolution)
diagnostics = append(diagnostics, resolved.diagnostics...)
if hasErrors(resolved.diagnostics) {
return resolvedLoad[T]{}, diagnostics
@@ -71,6 +68,7 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
final = resolved.final
resolutionSnapshots = resolved.snapshots
applyResolutionMetadata(&metadata, resolved)
referenceInput = documentWithMetadataLocations(final, metadata, input.document)
}
applications := []historyApplication(nil)
if input.composition != nil {
@@ -82,7 +80,13 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
if input.versionState != nil {
versionEvidence = append(versionEvidence, input.versionState.evidence...)
}
return resolvedLoad[T]{configuration: configuration, decoded: decoded, metadata: metadata, final: final, snapshots: resolutionSnapshots, applications: applications, versionEvidence: versionEvidence, inheritance: input.inheritance, contexts: input.contexts}, diagnostics
return resolvedLoad[T]{
configuration: configuration, decoded: decoded, metadata: metadata, final: final,
snapshots: resolutionSnapshots, applications: applications,
postDecodeApps: cloneHistoryApplications(prepared.postDecodeApps), structuralInput: document.Clone(prepared.structuralInput),
preCLI: document.Clone(prepared.preCLI), preCLIOrigins: cloneOrigins(prepared.preCLIOrigins), referenceInput: referenceInput,
versionEvidence: versionEvidence, inheritance: input.inheritance, contexts: input.contexts,
}, diagnostics
}
func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, mode secretCloneMode) (T, *Diagnostic) {
@@ -105,12 +109,15 @@ func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T]
}
decoded, metadata := resolved.decoded, resolved.metadata
final, applications, resolutionSnapshots := resolved.final, resolved.applications, resolved.snapshots
finalizeMetadataPaths(&metadata, final, applications, decoded.baseline, decoded.final)
secrets := classifySecretPathsWithEvidence(schema, definition.Secret, metadata, final, applications, resolved.versionEvidence, decoded.baseline, decoded.final)
structuralInput := document.Clone(resolved.structuralInput)
preCLI := document.Clone(resolved.preCLI)
allApplications := append(cloneHistoryApplications(applications), cloneHistoryApplications(resolved.postDecodeApps)...)
finalizeMetadataPaths(&metadata, final, allApplications, decoded.baseline, resolved.structuralInput, resolved.preCLI, decoded.final)
secrets := classifySecretPathsWithEvidence(schema, definition.Secret, metadata, final, allApplications, resolved.versionEvidence, decoded.baseline, resolved.structuralInput, resolved.preCLI, decoded.final)
sources := sourceInfosWithVersionEvidence(accumulator.sources, resolved.versionEvidence, secrets, false)
retainedTrees := make([]*document.Node, 0, len(applications)+2)
retainedTrees = append(retainedTrees, decoded.final, final)
for _, application := range applications {
retainedTrees := make([]*document.Node, 0, len(allApplications)+5)
retainedTrees = append(retainedTrees, decoded.final, final, structuralInput, preCLI)
for _, application := range allApplications {
retainedTrees = append(retainedTrees, application.value)
}
inspection := &Inspection{
@@ -121,6 +128,10 @@ func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T]
baseline: decoded.baseline,
input: decoded.final,
inputOrigins: cloneOriginIndex(decoded.origins),
structuralInput: structuralInput,
preCLI: preCLI,
preCLIOrigins: cloneOrigins(resolved.preCLIOrigins),
postDecodeApps: cloneHistoryApplications(resolved.postDecodeApps),
final: final,
immutableLookup: buildDocumentLookup(retainedTrees...),
resolution: resolutionSnapshots,

View File

@@ -0,0 +1,79 @@
package engine
import (
"fmt"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestMetadataLocationsAcrossBroadAuthoredAndDefaultMappings(t *testing.T) {
type config map[string]map[string]string
defaults := config{}
var body strings.Builder
for index := range 96 {
name := fmt.Sprintf("authored-%03d", index)
fmt.Fprintf(&body, "%s: {present: source}\n", name)
defaults[name] = map[string]string{"absent": "default"}
defaults[fmt.Sprintf("default-%03d", index)] = map[string]string{"absent": "default"}
}
input := parseDocumentForDecode(t, body.String())
decoded, issues := decodeConfiguration(mustSchemaFor[config](t), reflect.ValueOf(defaults), input)
if len(issues) != 0 {
t.Fatal(issues)
}
located := documentWithMetadataLocations(decoded.final, metadataFromDecoded(decoded), input)
inputIndex := buildDocumentLookup(input)
locatedIndex := buildDocumentLookup(located)
for index := range 96 {
authored := Keys(fmt.Sprintf("authored-%03d", index))
mapping, _ := documentAtPathWithLookup(input, authored, inputIndex)
present, _ := documentAtPathWithLookup(input, authored.Key("present"), inputIndex)
for _, test := range []struct {
path Path
want document.Location
}{
{authored.Key("present"), present.Location},
{authored.Key("absent"), mapping.Location},
{Keys(fmt.Sprintf("default-%03d", index), "absent"), input.Location},
} {
node, found := documentAtPathWithLookup(located, test.path, locatedIndex)
if !found || node.Location != test.want {
t.Fatalf("location at %s = %#v, want %#v", test.path, node, test.want)
}
}
}
walkDocument(located, Path{}, func(_ Path, node *document.Node) {
for _, entry := range node.Mapping {
if entry.KeyLocation != entry.Value.Location || entry.ValueLocation != entry.Value.Location {
t.Fatal("entry and value locations disagree")
}
}
})
before := decoded.final.Mapping[0].Value.Mapping[0].Value.Text
located.Mapping[0].Value.Mapping[0].Value.Text = "changed snapshot"
if decoded.final.Mapping[0].Value.Mapping[0].Value.Text != before || input.Mapping[0].Value.Mapping[0].Value.Text != "source" {
t.Fatal("location projection changed a borrowed input")
}
}
func TestIndexedNearestLocationsPreserveMissingAndSyntheticCoordinates(t *testing.T) {
root := parseDocumentForDecode(t, "mapping: {'0': value}\nitems: [{value: source}]\nscalar: stop\n")
lookup := buildDocumentLookup(root)
for _, path := range []Path{Path{}, Keys("missing"), Keys("mapping", "missing"), Keys("mapping", "0"), Keys("mapping").Index(0), Keys("items").Index(4), Keys("items").Index(0).Key("missing"), Keys("scalar", "child"), Path{}.Index(-1)} {
want, wantOK := nearestDocumentLocation(root, path)
got, gotOK := nearestDocumentLocationWithLookup(root, path, lookup)
if got != want || gotOK != wantOK {
t.Fatalf("nearest location changed for %s: %#v/%t, want %#v/%t", path, got, gotOK, want, wantOK)
}
}
final := parseDocumentForDecode(t, "value: cli\n")
synthetic := Location{SourceID: "cli-1"}
metadata := Metadata{fields: map[string]FieldInfo{"/value": {Origin: Origin{Layer: LayerCLI, Locations: []Location{synthetic}}}}}
located := documentWithMetadataLocations(final, metadata, root)
if location := located.Mapping[0].Value.Location; location.SourceID != "cli-1" || location.Line != 0 || location.Column != 0 {
t.Fatalf("synthetic CLI location was replaced by fallback: %#v", location)
}
}

View File

@@ -31,11 +31,18 @@ func selectOptions[T any](source Source, supplied []Options[T]) (Options[T], []D
if options.Filesystem != nil {
diagnostics = append(diagnostics, filesystemOptionDiagnostics(source, options.Filesystem)...)
}
if source.kind == sourceNone && options.Imports {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource cannot enable imports"))
if source.kind == sourceSearch {
for _, issue := range source.searchIssues {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, issue))
}
if source.search == nil {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "configuration search is invalid"))
} else {
diagnostics = append(diagnostics, validateSearch(*source.search, source.searchApp)...)
}
}
if source.kind == sourceNone && options.Overlay != "" && options.Overlay != "default" {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource cannot select an active overlay"))
if source.kind == sourceNone && activeOverlay(options.Overlay) && options.Filesystem == nil {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "NoSource overlays require explicit filesystem context"))
}
if options.Filesystem == nil && (source.kind == sourceBytes || source.kind == sourceReader || source.kind == sourceMapping) && (options.Imports || activeOverlay(options.Overlay)) {
diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "synthetic-source imports or overlays require filesystem context"))

View File

@@ -0,0 +1,210 @@
package engine
import (
"errors"
"reflect"
"strings"
"testing"
)
type ordinarySequenceItem struct {
ID string `yaml:"id"`
}
type ordinarySequenceRecord struct {
ID string `yaml:"id"`
Children []ordinarySequenceItem `yaml:"children"`
}
type ordinarySequenceConfig struct {
Items []ordinarySequenceItem `yaml:"items"`
Catalog struct {
Items []ordinarySequenceItem `yaml:"items"`
} `yaml:"catalog"`
Records []ordinarySequenceRecord `yaml:"records"`
}
func TestOrdinaryLoadAcceptsDeclarationsWithoutImports(t *testing.T) {
result, err := testLoad[ordinarySequenceConfig](Bytes("catalog.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
items:
- id: prompt
`)))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(result.Config.Items, []ordinarySequenceItem{{ID: "prompt"}}) {
t.Fatalf("items = %#v", result.Config.Items)
}
if _, ok := result.Metadata().Lookup(Keys("configkit")); ok {
t.Fatal("collection controls reached application metadata")
}
items, ok := result.Metadata().Lookup(Keys("items"))
if !ok || !items.Explicit || len(items.Origin.Locations) != 1 {
t.Fatalf("items metadata = %#v, %v", items, ok)
}
}
func TestOrdinaryLoadKeepsDisabledImportRejection(t *testing.T) {
_, err := testLoad[ordinarySequenceConfig](Bytes("disabled.yml", []byte(`configkit:
imports: [{path: fragment.yml}]
sequences: [{destination: [items], key: id}]
`)))
if err == nil {
t.Fatal("source with disabled imports succeeded")
}
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeImportDisabled {
t.Fatalf("error = %#v", err)
}
}
func TestOrdinaryLoadValidatesSingletonCollectionIdentity(t *testing.T) {
_, err := testLoad[ordinarySequenceConfig](Bytes("missing-identity.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
items: [{}]
`)))
if err == nil {
t.Fatal("singleton sequence without identity succeeded")
}
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 || loadError.Diagnostics()[0].Code != CodeComposeInvalid {
t.Fatalf("error = %#v", err)
}
}
func TestOrdinaryLoadCombinesImportedCollectionPeers(t *testing.T) {
tests := []struct {
name string
second string
want []ordinarySequenceItem
wantCode Code
}{
{name: "distinct identities", second: "items: [{id: second}]\n", want: []ordinarySequenceItem{{ID: "first"}, {ID: "second"}}},
{name: "duplicate peer identity", second: "items: [{id: first}]\n", wantCode: CodeComposeConflict},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: first.yml
- path: second.yml
sequences:
- destination: [items]
key: id
mode: append
`)
writeImportFile(t, directory, "first.yml", "items: [{id: first}]\n")
writeImportFile(t, directory, "second.yml", test.second)
result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Imports: true})
if test.wantCode != "" {
if result != nil || err == nil {
t.Fatalf("result, err = %#v, %v", result, err)
}
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) == 0 || loadError.Diagnostics()[0].Code != test.wantCode {
t.Fatalf("error = %#v", err)
}
return
}
if err != nil {
var loadError *LoadError
if errors.As(err, &loadError) {
t.Fatalf("diagnostics = %#v", loadError.Diagnostics())
}
t.Fatal(err)
}
if !reflect.DeepEqual(result.Config.Items, test.want) {
t.Fatalf("items = %#v, want %#v", result.Config.Items, test.want)
}
inline, err := testLoad[ordinarySequenceConfig](Bytes("inline.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
mode: append
items: [{id: first}, {id: second}]
`)))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(result.Config.Items, inline.Config.Items) {
t.Fatalf("imported items %#v differ from inline items %#v", result.Config.Items, inline.Config.Items)
}
})
}
}
func TestImportedCollectionDeclarationsUseRelativePlacement(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: first.yml
destination: [catalog]
- path: second.yml
destination: [catalog]
`)
fragment := `configkit:
sequences:
- destination: [items]
key: id
mode: append
items: [{id: ITEM}]
`
writeImportFile(t, directory, "first.yml", strings.ReplaceAll(fragment, "ITEM", "first"))
writeImportFile(t, directory, "second.yml", strings.ReplaceAll(fragment, "ITEM", "second"))
result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Imports: true})
if err != nil {
t.Fatal(err)
}
want := []ordinarySequenceItem{{ID: "first"}, {ID: "second"}}
if !reflect.DeepEqual(result.Config.Catalog.Items, want) {
t.Fatalf("catalog items = %#v, want %#v", result.Config.Catalog.Items, want)
}
}
func TestSequenceItemSourceAppliesItsCollectionDeclarations(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [records]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", `configkit:
sequences:
- destination: [children]
key: id
mode: append
id: parent
children: [{id: child}]
`)
result, err := testLoad[ordinarySequenceConfig](File(mainPath), Options[ordinarySequenceConfig]{Imports: true})
if err != nil {
t.Fatal(err)
}
want := []ordinarySequenceRecord{{ID: "parent", Children: []ordinarySequenceItem{{ID: "child"}}}}
if !reflect.DeepEqual(result.Config.Records, want) {
t.Fatalf("records = %#v, want %#v", result.Config.Records, want)
}
}
func TestOrdinaryLoadAllowsDeclarationsWithoutValues(t *testing.T) {
result, err := testLoad[ordinarySequenceConfig](Bytes("policy.yml", []byte(`configkit:
sequences:
- destination: [items]
key: id
`)))
if err != nil {
t.Fatal(err)
}
if result.Config.Items != nil {
t.Fatalf("items = %#v, want absent", result.Config.Items)
}
}

View File

@@ -52,8 +52,221 @@ func (a *assemblyAccumulator) applyOverlay(overlay *document.Node, incomingOrigi
return nil
}
// applyOverlayWithCollectionPolicy is the shared recursive-precedence entry for
// overlays. The caller supplies only the policy and declarations owned by this
// contribution; layered loading can use the same operation when its lifecycle
// reaches this boundary.
func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy(
overlay *document.Node,
incomingOrigins map[string]Origin,
incomingPolicies collectionPolicyScope,
applications []historyApplication,
layer Layer,
sequencePolicy sequencePolicy,
declarations []sequenceDeclarationSpec,
) []Diagnostic {
_, diagnostics := a.applyOverlayWithCollectionProjection(overlay, incomingOrigins, incomingPolicies, applications, layer, sequencePolicy, declarations)
return diagnostics
}
// applyOverlayWithCollectionProjection also returns the incoming tier's item
// assignments so acquisition evidence can follow the same operation as values.
func (a *assemblyAccumulator) applyOverlayWithCollectionProjection(
overlay *document.Node,
incomingOrigins map[string]Origin,
incomingPolicies collectionPolicyScope,
applications []historyApplication,
layer Layer,
sequencePolicy sequencePolicy,
declarations []sequenceDeclarationSpec,
) ([]sequenceHistoryContribution, []Diagnostic) {
if a == nil || a.current == nil || a.current.root == nil || a.current.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping {
return nil, invalidOverlayDocument()
}
mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer))
sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, incomingPolicies, layer, sequencePolicy, declarations)
if len(diagnostics) != 0 {
return nil, diagnostics
}
if len(applications) == 0 {
application := historyApplication{layer: layer, value: overlay, sequences: sequences}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
a.current.applications = append(a.current.applications, application)
} else {
for _, application := range projectTierHistoryApplications(applications, sequences) {
application.layer = layer
a.current.applications = append(a.current.applications, application)
}
}
return sequences, nil
}
func mergeOverlayMappingsWithCollectionPolicy(
assembly *assemblyAccumulator,
overlay *document.Node,
incomingOrigins map[string]Origin,
incomingPolicies collectionPolicyScope,
layer Layer,
sequencePolicy sequencePolicy,
declarations []sequenceDeclarationSpec,
) ([]sequenceHistoryContribution, []Diagnostic) {
result := assembly.current
claims := compositionClaims{byPath: make(map[string][]compositionClaim)}
claims.append(overlay, Path{}, 0)
identityFieldsByPath, resolvedPolicies, diagnostics := resolveStrictCollectionPolicies(
claims.ordered, sequencePolicy, result.collectionPolicies, declarations,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
declarationsByPath := make(map[string][]sequenceDeclarationSpec, len(declarations))
registeredPaths := make(map[string]Path)
for _, declaration := range declarations {
key := declaration.destination.String()
declarationsByPath[key] = append(declarationsByPath[key], declaration)
registeredPaths[key] = declaration.destination.clone()
}
registered := make(map[string]bool)
for _, claim := range claims.ordered {
if claim.kind == document.Sequence {
if _, ok := sequencePolicy.identityFieldsAt(claim.path); ok {
registered[claim.path.String()] = true
registeredPaths[claim.path.String()] = claim.path.clone()
}
}
}
for key := range declarationsByPath {
registered[key] = true
}
var overlayLookup *documentLookup
if len(result.collectionPolicies.collections) != 0 {
overlayLookup = buildDocumentLookup(overlay)
}
for key := range result.collectionPolicies.collections {
if node, found := documentAtPathWithLookup(overlay, result.collectionPolicies.collections[key].destination, overlayLookup); found && node != nil {
registered[key] = true
registeredPaths[key] = result.collectionPolicies.collections[key].destination.clone()
}
}
resolutions := make(map[string]collectionPolicyResolution, len(registered))
for key := range registered {
path := registeredPaths[key]
policy, hasPolicy := resolvedPolicies.at(path)
if !hasPolicy {
continue
}
operation := sequenceModeReplace
if policy.explicitModePresent {
operation = policy.explicitMode
} else if len(identityFieldsByPath[key]) != 0 {
operation = sequenceModeUpsert
}
resolutions[key] = collectionPolicyResolution{
policy: policy, operation: operation,
effectiveIdentityFields: cloneStrings(identityFieldsByPath[key]),
}
}
sequenceApplications := make(map[string]sequenceHistoryContribution)
policyValues := make(map[string]bool)
var combinationDiagnostics []Diagnostic
mergeOverlayMappingsWithHooks(result.root, overlay, Path{}, &assembly.index,
recursiveMergeHooks{
added: func(path Path, incoming *document.Node) {
collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer)
},
combined: func(path Path, incoming *document.Node) {
mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, incoming, layer))
},
replaced: func(path Path, existing, incoming *document.Node) {
assembly.current.collectionPolicies.retireCollectionPoliciesAtOrBelow(path)
recordOverlayReplacement(result, path, existing, incoming, layer, true, incomingOrigins)
},
merge: func(path Path, existing, incoming *document.Node) (*document.Node, bool) {
resolution, active := resolutions[path.String()]
if !active {
return nil, false
}
combined, issues := combineSequence(existing, incoming, resolution, sequenceCombinationContext{destination: path})
if len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
return existing, false
}
applyOverlaySequenceEvidence(result, path, existing, incoming, combined, resolution, incomingOrigins, incomingPolicies, layer)
policyValues[path.String()] = true
history := sequenceHistoryContribution{path: path.clone(), mode: sequenceHistoryMapped}
if resolution.operation == sequenceModeReplace {
history.mode = sequenceHistoryReplace
if incoming != nil && incoming.Kind == document.Sequence {
history.length = len(incoming.Sequence)
}
} else {
history.items = append([]sequenceItemProjection(nil), combined.higherItems...)
history.replacements = append([]sequenceItemReplacement(nil), combined.replacements...)
}
sequenceApplications[path.String()] = history
return combined.value, true
},
},
)
if len(combinationDiagnostics) != 0 {
return nil, canonicalDiagnostics(combinationDiagnostics)
}
// Values are now complete. Index lazily for declaration-only checks;
// this lookup is never retained across the value-changing merge above.
var retainedLookup *documentLookup
for key, grouped := range declarationsByPath {
if policyValues[key] {
continue
}
path := grouped[0].destination
applicationIdentity, _ := sequencePolicy.identityFieldsAt(path)
current, diagnostics := resolveCollectionPolicy(path, grouped, applicationIdentity, nil)
if len(diagnostics) != 0 {
combinationDiagnostics = append(combinationDiagnostics, diagnostics...)
continue
}
lower, found := assembly.current.collectionPolicies.at(path)
var lowerPolicy *collectionPolicy
if found {
lowerPolicy = &lower
}
if retainedLookup == nil {
retainedLookup = buildDocumentLookup(assembly.current.root)
}
retained, _ := documentAtPathWithLookup(assembly.current.root, path, retainedLookup)
if retained != nil && retained.Kind == document.Sequence && len(current.effectiveIdentityFields) != 0 {
if _, issues := validateSequenceIdentity(retained, path, current.effectiveIdentityFields, sequenceIdentityDiagnosticCodes{invalid: CodeSequenceInvalid, conflict: CodeSequenceConflict}); len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
continue
}
}
policy, issues := collectionPolicyWithoutValue(lowerPolicy, current.policy, retained, path)
if len(issues) != 0 {
combinationDiagnostics = append(combinationDiagnostics, issues...)
continue
}
assembly.current.collectionPolicies.set(path, policy)
}
if len(combinationDiagnostics) != 0 {
return nil, canonicalDiagnostics(combinationDiagnostics)
}
keys := make([]string, 0, len(sequenceApplications))
for key := range sequenceApplications {
keys = append(keys, key)
}
slices.Sort(keys)
sequences := make([]sequenceHistoryContribution, 0, len(keys))
for _, key := range keys {
sequences = append(sequences, sequenceApplications[key])
}
return sequences, nil
}
func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool, incomingOrigins map[string]Origin, index *mappingIndex) {
mergeRecursiveMappings(target, overlay, current, index, recursiveMergeHooks{
mergeOverlayMappingsWithHooks(target, overlay, current, index, recursiveMergeHooks{
added: func(path Path, incoming *document.Node) {
collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer)
},
@@ -78,22 +291,165 @@ func mergeOverlayMappings(result *composedDocument, target, overlay *document.No
})
}
func mergeOverlayMappingsWithHooks(
target, overlay *document.Node,
current Path,
index *mappingIndex,
hooks recursiveMergeHooks,
) {
mergeRecursiveMappings(target, overlay, current, index, hooks)
}
func recordOverlayReplacement(
result *composedDocument,
path Path,
existing, incoming *document.Node,
layer Layer,
recordReplacement bool,
incomingOrigins map[string]Origin,
) {
var displaced []displacedDefinition
var previous Origin
if existing != nil {
displaced = displacedOrigins(existing, result.origins, path)
previous = cloneOrigin(result.origins[path.String()])
deleteOriginSubtree(result.origins, path, existing)
}
collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming, path, layer)
if recordReplacement && existing != nil {
result.overlays = append(result.overlays, overlayRecord{
change: Change{
Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, incoming, layer),
Previous: previousOrigins(previous),
},
displaced: displaced,
})
}
}
func applyOverlaySequenceEvidence(
result *composedDocument,
path Path,
lower, higher *document.Node,
combined sequenceCombinationResult,
resolution collectionPolicyResolution,
incomingOrigins map[string]Origin,
incomingPolicies collectionPolicyScope,
layer Layer,
) {
lowerPolicy, hadLowerPolicy := result.collectionPolicies.at(path)
if resolution.operation == sequenceModeReplace {
var lowerValue *collectionPolicy
if hadLowerPolicy {
lowerValue = &lowerPolicy
}
result.collectionPolicies.retireCollectionPoliciesAtOrBelow(path)
resolution.policy = collectionPolicyForReplacement(lowerValue, resolution.policy)
}
if combined.value == nil || combined.value.Kind != document.Sequence {
recordOverlayReplacement(result, path, lower, higher, layer, true, incomingOrigins)
result.collectionPolicies.set(path, resolution.policy)
return
}
if resolution.operation == sequenceModeReplace {
var displaced []displacedDefinition
var previous Origin
if lower != nil {
displaced = displacedOrigins(lower, result.origins, path)
previous = cloneOrigin(result.origins[path.String()])
deleteOriginSubtree(result.origins, path, lower)
}
mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, higher, layer))
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
source := higher.Sequence[projection.inputIndex]
collectOverlayOriginsAt(result.origins, incomingOrigins, source, path.Index(projection.inputIndex), path.Index(projection.outputIndex), layer)
}
}
if lower != nil {
result.overlays = append(result.overlays, overlayRecord{
change: Change{
Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, higher, layer),
Previous: previousOrigins(previous),
}, displaced: displaced,
})
}
} else {
mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, higher, layer))
for _, replacement := range combined.replacements {
lowerPath := path.Index(replacement.lowerIndex)
higherPath := path.Index(replacement.higherIndex)
outputPath := path.Index(replacement.outputIndex)
if lower == nil || lower.Kind != document.Sequence || replacement.lowerIndex < 0 || replacement.lowerIndex >= len(lower.Sequence) ||
higher == nil || higher.Kind != document.Sequence || replacement.higherIndex < 0 || replacement.higherIndex >= len(higher.Sequence) {
continue
}
lowerItem := lower.Sequence[replacement.lowerIndex]
higherItem := higher.Sequence[replacement.higherIndex]
displaced := displacedOrigins(lowerItem, result.origins, lowerPath)
previous := cloneOrigin(result.origins[lowerPath.String()])
deleteOriginSubtree(result.origins, lowerPath, lowerItem)
collectOverlayOriginsAt(result.origins, incomingOrigins, higherItem, higherPath, outputPath, layer)
result.overlays = append(result.overlays, overlayRecord{
change: Change{
Path: outputPath.clone(), Origin: incomingOrigin(incomingOrigins, higherPath, higherItem, layer),
Previous: previousOrigins(previous),
},
displaced: displaced,
})
}
for _, projection := range combined.higherItems {
if higher != nil && higher.Kind == document.Sequence && projection.inputIndex >= 0 && projection.inputIndex < len(higher.Sequence) {
higherItem := higher.Sequence[projection.inputIndex]
collectOverlayOriginsAt(result.origins, incomingOrigins, higherItem, path.Index(projection.inputIndex), path.Index(projection.outputIndex), layer)
}
}
}
policy := cloneCollectionPolicy(resolution.policy)
policy.recordOrder = nil
policy.recordPolicies = nil
lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems)
higherProjection := newSequenceItemProjectionIndex(combined.higherItems)
incomingCollection, hasIncomingCollection := incomingPolicies.at(path)
for outputIndex := range combined.value.Sequence {
if higherIndex, replaced := higherProjection.inputForOutput(outputIndex); replaced {
if record, found := incomingCollection.recordPolicyAt(higherIndex); hasIncomingCollection && found {
policy.setRecordPolicyAt(outputIndex, retainedCollectionRecordPolicy(record))
} else {
policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
}
continue
}
if hadLowerPolicy && resolution.operation != sequenceModeReplace {
if lowerIndex, retained := lowerProjection.inputForOutput(outputIndex); retained {
if record, found := lowerPolicy.recordPolicyAt(lowerIndex); found {
policy.setRecordPolicyAt(outputIndex, retainedCollectionRecordPolicy(record))
}
}
} else {
policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
}
}
result.collectionPolicies.set(path, policy)
}
func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) {
collectOverlayOriginsFrom(origins, nil, node, path, layer)
}
func collectOverlayOriginsFrom(origins, incomingOrigins map[string]Origin, node *document.Node, path Path, layer Layer) {
origins[path.String()] = incomingOrigin(incomingOrigins, path, node, layer)
switch node.Kind {
case document.Mapping:
for _, entry := range node.Mapping {
collectOverlayOriginsFrom(origins, incomingOrigins, entry.Value, path.Key(entry.Key), layer)
}
case document.Sequence:
for index, child := range node.Sequence {
collectOverlayOriginsFrom(origins, incomingOrigins, child, path.Index(index), layer)
}
}
collectOverlayOriginsAt(origins, incomingOrigins, node, path, path, layer)
}
func collectOverlayOriginsAt(origins, incomingOrigins map[string]Origin, node *document.Node, from, to Path, layer Layer) {
walkDocument(node, Path{}, func(relative Path, value *document.Node) {
source := appendDestination(from, relative)
destination := appendDestination(to, relative)
origins[destination.String()] = incomingOrigin(incomingOrigins, source, value, layer)
})
}
func mergeOverlayOrigin(origins map[string]Origin, path Path, location document.Location, layer Layer) {

View File

@@ -19,9 +19,28 @@ type preparedOverride struct {
nodes int
}
func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) ([]preparedOverride, []SourceInfo, []Diagnostic) {
// preparedOverrideBatch owns all override values and synthetic source records
// needed to apply one CLI batch after its preparation checks succeed.
type preparedOverrideBatch struct {
overrides []preparedOverride
sources []SourceInfo
}
type overridePreparationInput struct {
root *document.Node
inputBytes int64
inputNodes int
}
// preparePostStructuralOverrides snapshots CLI input for typed assignment after
// structural expansion, when inheritance directives can no longer be supplied.
func preparePostStructuralOverrides(base overridePreparationInput, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (preparedOverrideBatch, []Diagnostic) {
return prepareOverrideBatch(base, sources, overrides, rule, limits)
}
func prepareOverrideBatch(base overridePreparationInput, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (preparedOverrideBatch, []Diagnostic) {
if len(overrides) > maxOverrides || len(sources)+len(overrides) > limits.sources {
return nil, cloneSourceInfos(sources), []Diagnostic{{
return preparedOverrideBatch{sources: cloneSourceInfos(sources)}, []Diagnostic{{
Code: CodeOverrideLimitExceeded, Severity: SeverityError,
Message: "configuration overrides exceed the record or source-count limit",
}}
@@ -37,7 +56,7 @@ func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []
}
}
if len(diagnostics) != 0 {
return nil, cloneSourceInfos(sources), canonicalDiagnostics(diagnostics)
return preparedOverrideBatch{sources: cloneSourceInfos(sources)}, canonicalDiagnostics(diagnostics)
}
prepared := make([]preparedOverride, len(overrides))
@@ -89,7 +108,7 @@ func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []
})
continue
}
if pathContainsDirective(override.Path) {
if pathContainsDirective(override.Path) || pathContainsInheritanceDirective(override.Path) {
diagnostics = append(diagnostics, Diagnostic{
Code: CodeOverrideDirective, Severity: SeverityError, Path: override.Path.clone(),
Message: "configuration overrides cannot create configuration directives", Primary: location,
@@ -106,7 +125,7 @@ func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []
diagnostics = append(diagnostics, diagnostic)
continue
}
if relative, ok := overrideDirectivePath(value, Path{}); ok {
if relative, ok := overrideStructuralDirectivePath(value, Path{}); ok {
diagnostics = append(diagnostics, Diagnostic{
Code: CodeOverrideDirective, Severity: SeverityError, Path: appendOverridePath(override.Path, relative),
Message: "configuration overrides cannot create configuration directives", Primary: location,
@@ -136,10 +155,10 @@ func prepareOverrides(base *composedDocument, sources []SourceInfo, overrides []
usedNodes += nodes
}
if len(diagnostics) != 0 {
return nil, allSources, canonicalDiagnostics(diagnostics)
return preparedOverrideBatch{sources: allSources}, canonicalDiagnostics(diagnostics)
}
return prepared, allSources, nil
return preparedOverrideBatch{overrides: prepared, sources: allSources}, nil
}
func validOverrideLabel(label string) bool {
@@ -191,26 +210,42 @@ func pathContainsDirective(path Path) bool {
return false
}
func pathContainsInheritanceDirective(path Path) bool {
for _, segment := range path.segments {
if segment.key == inheritanceKey {
return true
}
}
return false
}
func appendOverridePath(base, relative Path) Path {
result := base.clone()
result.segments = append(result.segments, clonePathSegments(relative.segments, 0)...)
return result
}
func overrideDirectivePath(node *document.Node, path Path) (Path, bool) {
func overrideStructuralDirectivePath(node *document.Node, path Path) (Path, bool) {
return overrideStructuralDirectivePathAt(node, path)
}
func overrideStructuralDirectivePathAt(node *document.Node, path Path) (Path, bool) {
if node == nil {
return Path{}, false
}
if node.Kind == document.Mapping {
for _, entry := range node.Mapping {
child := path.Key(entry.Key)
if entry.Key == directiveKey {
if entry.Key == directiveKey || entry.Key == inheritanceKey {
return child, true
}
if found, ok := overrideDirectivePath(entry.Value, child); ok {
if found, ok := overrideStructuralDirectivePathAt(entry.Value, child); ok {
return found, true
}
}
} else if node.Kind == document.Sequence {
for index, child := range node.Sequence {
if found, ok := overrideDirectivePath(child, path.Index(index)); ok {
if found, ok := overrideStructuralDirectivePathAt(child, path.Index(index)); ok {
return found, true
}
}

View File

@@ -1,110 +0,0 @@
package engine
import (
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func applyOverrides(base *composedDocument, sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
if len(overrides) == 0 {
return base, cloneSourceInfos(sources), nil
}
assembly := newAssemblyAccumulator(cloneComposedDocument(base))
allSources, diagnostics := assembly.applyOverrides(sources, overrides, rule, limits)
if len(diagnostics) != 0 {
return nil, allSources, diagnostics
}
return assembly.current, allSources, nil
}
func (a *assemblyAccumulator) applyOverrides(sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) {
if len(overrides) == 0 {
return cloneSourceInfos(sources), nil
}
prepared, allSources, diagnostics := prepareOverrides(a.current, sources, overrides, rule, limits)
if len(diagnostics) != 0 {
return allSources, diagnostics
}
for _, override := range prepared {
applyPreparedOverride(a.current, override, &a.index)
a.current.inputBytes += override.bytes
a.current.inputNodes += override.nodes
}
return allSources, nil
}
func applyPreparedOverride(result *composedDocument, override preparedOverride, index *mappingIndex) {
location := document.Location{SourceID: override.source.ID}
mergeOverlayOrigin(result.origins, Path{}, location, LayerCLI)
current := result.root
currentPath := Path{}
for segmentIndex, segment := range override.path.segments {
path := currentPath.Key(segment.key)
position := index.find(current, segment.key)
if segmentIndex < len(override.path.segments)-1 {
if position < 0 {
child := &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{}}
index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child})
collectOverlayOrigins(result.origins, child, path, LayerCLI)
current = child
} else {
current = current.Mapping[position].Value
mergeOverlayOrigin(result.origins, path, location, LayerCLI)
}
currentPath = path
continue
}
previous := cloneOrigin(result.origins[path.String()])
var displaced []displacedDefinition
if position < 0 {
incoming := document.Clone(override.value)
index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming})
collectOverlayOrigins(result.origins, incoming, path, LayerCLI)
} else {
existing := current.Mapping[position].Value
displaced = affectedOverrideOrigins(existing, override.value, result.origins, path, index)
if existing.Kind == document.Mapping && override.value.Kind == document.Mapping {
mergeOverlayOrigin(result.origins, path, location, LayerCLI)
mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false, nil, index)
} else {
deleteOriginSubtree(result.origins, path, existing)
current.Mapping[position] = document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: document.Clone(override.value)}
collectOverlayOrigins(result.origins, current.Mapping[position].Value, path, LayerCLI)
}
}
result.overlays = append(result.overlays, overlayRecord{change: Change{
Path: path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous),
}, displaced: displaced})
result.applications = append(result.applications, historyApplication{
layer: LayerCLI, path: path.clone(), value: override.value,
origin: overlayOrigin(LayerCLI, location),
})
}
}
func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) []displacedDefinition {
var displaced []displacedDefinition
collectAffectedOverrideOrigins(&displaced, existing, incoming, origins, path, index)
return sortDisplacedDefinitions(displaced)
}
func collectAffectedOverrideOrigins(displaced *[]displacedDefinition, existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) {
if existing.Kind != document.Mapping || incoming.Kind != document.Mapping {
collectDisplacedOrigins(displaced, existing, origins, path, Origin{})
return
}
*displaced = append(*displaced, displacedDefinition{path: path.clone(), origin: cloneOrigin(origins[path.String()])})
for _, entry := range incoming.Mapping {
position := index.find(existing, entry.Key)
if position < 0 {
continue
}
childPath := path.Key(entry.Key)
childExisting := existing.Mapping[position].Value
if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping {
collectAffectedOverrideOrigins(displaced, childExisting, entry.Value, origins, childPath, index)
} else {
collectDisplacedOrigins(displaced, childExisting, origins, childPath, Origin{})
}
}
}

View File

@@ -0,0 +1,246 @@
package engine
import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
// postDecodeOverrideContext keeps the structural, typed pre-CLI, and effective
// post-CLI views distinct while preparing the existing inspection machinery.
type postDecodeOverrideContext struct {
structuralInput *document.Node
preCLI *document.Node
preCLIOrigins map[string]Origin
decoded decodedConfiguration
metadata Metadata
resolution resolutionInput
sources []SourceInfo
applications []historyApplication
postDecodeApps []historyApplication
referenceInput *document.Node
}
func preparePostDecodeOverrideContext(input assembledInput, before decodedConfiguration, assigned decodedOverrideResult) postDecodeOverrideContext {
after := before
after.value = assigned.value
after.presence = cloneInputPresence(assigned.presence)
after.origins = cloneOrigins(assigned.origins)
after.final = document.Clone(assigned.final)
metadata := metadataFromDecoded(before)
cliRecords := overrideMetadataRecords(before, assigned.overrides)
var structuralOverlays []overlayRecord
if input.inheritance != nil && input.composition != nil {
structuralOverlays = input.composition.overlays
applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, structuralOverlays, input.inheritance)
} else if input.composition != nil {
structuralOverlays = input.composition.overlays
applyOverlayMetadata(&metadata, structuralOverlays)
}
if len(cliRecords) != 0 {
metadata.fields = make(map[string]FieldInfo)
collectFieldInfo(after.final, Path{}, after, metadata.fields)
appendPostDecodeCLIHistory(&metadata, cliRecords)
}
sources := cloneSourceInfos(input.sources)
knownSources := make(map[string]bool, len(sources)+len(assigned.overrides))
for _, source := range sources {
knownSources[source.ID] = true
}
for _, override := range assigned.overrides {
if !knownSources[override.source.ID] {
sources = append(sources, cloneSourceInfo(override.source))
knownSources[override.source.ID] = true
}
}
if input.versionState != nil {
sources = sourceInfosWithVersionEvidence(sources, input.versionState.evidence, nil, true)
}
resolution := newResolutionInput(metadata, assigned.final, sources, input.contexts)
applications := structuralHistoryApplications(input)
var postDecodeApps []historyApplication
for _, override := range assigned.overrides {
postDecodeApps = append(postDecodeApps, historyApplication{
layer: LayerCLI,
path: override.path.clone(),
value: document.Clone(override.value),
origin: overlayOrigin(LayerCLI, document.Location{SourceID: override.source.ID}),
})
}
allApplications := append(cloneHistoryApplications(applications), cloneHistoryApplications(postDecodeApps)...)
finalizeMetadataPaths(&metadata, assigned.final, allApplications, input.document, before.final)
return postDecodeOverrideContext{
structuralInput: document.Clone(input.document),
preCLI: document.Clone(before.final),
preCLIOrigins: cloneOrigins(before.origins),
decoded: after,
metadata: metadata,
resolution: resolution,
sources: sources,
applications: applications,
postDecodeApps: postDecodeApps,
referenceInput: documentWithMetadataLocations(assigned.final, metadata, input.document),
}
}
// Source history is already assembled, with inheritance projected at query time.
// Reuse overlay event construction for CLI changes without reapplying displaced
// source definitions after those later assignments.
func appendPostDecodeCLIHistory(metadata *Metadata, records []overlayRecord) {
cli := Metadata{fields: metadata.fields}
applyOverlayMetadata(&cli, records)
for pointer, history := range cli.history {
for _, change := range history {
if change.Origin.Layer == LayerCLI {
appendInputChangeEvent(metadata.history, nil, pointer, change)
}
}
}
}
func overrideMetadataRecords(before decodedConfiguration, overrides []preparedOverride) []overlayRecord {
if len(overrides) == 0 {
return nil
}
lookup := buildDocumentLookup(before.final)
records := make([]overlayRecord, 0, len(overrides))
for _, override := range overrides {
location := document.Location{SourceID: override.source.ID}
previous := cloneOrigin(before.origins[override.path.String()])
var displaced []displacedDefinition
if existing, ok := documentAtPathWithLookup(before.final, override.path, lookup); ok {
displaced = affectedOverrideOrigins(existing, override.value, before.origins, override.path, new(mappingIndex))
}
records = append(records, overlayRecord{change: Change{
Path: override.path.clone(), Origin: overlayOrigin(LayerCLI, location), Previous: previousOrigins(previous),
}, displaced: displaced})
}
return records
}
func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) []displacedDefinition {
var displaced []displacedDefinition
collectAffectedOverrideOrigins(&displaced, existing, incoming, origins, path, index)
return sortDisplacedDefinitions(displaced)
}
func collectAffectedOverrideOrigins(displaced *[]displacedDefinition, existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) {
if existing.Kind != document.Mapping || incoming.Kind != document.Mapping {
collectDisplacedOrigins(displaced, existing, origins, path, Origin{})
return
}
*displaced = append(*displaced, displacedDefinition{path: path.clone(), origin: cloneOrigin(origins[path.String()])})
for _, entry := range incoming.Mapping {
position := index.find(existing, entry.Key)
if position < 0 {
continue
}
childPath := path.Key(entry.Key)
childExisting := existing.Mapping[position].Value
if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping {
collectAffectedOverrideOrigins(displaced, childExisting, entry.Value, origins, childPath, index)
} else {
collectDisplacedOrigins(displaced, childExisting, origins, childPath, Origin{})
}
}
}
func (context postDecodeOverrideContext) historyApplications() []historyApplication {
applications := cloneHistoryApplications(context.applications)
return append(applications, cloneHistoryApplications(context.postDecodeApps)...)
}
func structuralHistoryApplications(input assembledInput) []historyApplication {
if input.composition != nil {
return cloneHistoryApplications(input.composition.applications)
}
if input.document == nil {
return nil
}
return []historyApplication{{layer: LayerBase, value: document.Clone(input.document)}}
}
// documentWithMetadataLocations attaches effective origin coordinates to a
// canonical typed projection for secret and validation diagnostics.
func documentWithMetadataLocations(root *document.Node, metadata Metadata, fallback ...*document.Node) *document.Node {
located := document.Clone(root)
if located == nil {
return nil
}
var fallbackRoot *document.Node
if len(fallback) != 0 {
fallbackRoot = fallback[0]
}
var lookup *documentLookup
fallbackLocation := func(path Path) (document.Location, bool) {
if lookup == nil {
lookup = buildDocumentLookup(fallbackRoot)
}
return nearestDocumentLocationWithLookup(fallbackRoot, path, lookup)
}
var visit func(*document.Node, Path)
visit = func(node *document.Node, path Path) {
pointer := path.String()
if info, exists := metadata.fields[pointer]; exists {
if location, ok := metadataFieldLocation(metadata, pointer, info.Origin); ok {
node.Location = location
} else if location, ok := fallbackLocation(path); ok {
node.Location = location
}
} else if location, ok := fallbackLocation(path); ok {
node.Location = location
}
switch node.Kind {
case document.Mapping:
for index := range node.Mapping {
entry := &node.Mapping[index]
childPath := path.Key(entry.Key)
if info, exists := metadata.fields[childPath.String()]; exists {
if location, ok := metadataFieldLocation(metadata, childPath.String(), info.Origin); ok {
entry.KeyLocation = location
entry.ValueLocation = location
} else if location, ok := fallbackLocation(childPath); ok {
entry.KeyLocation = location
entry.ValueLocation = location
}
} else if location, ok := fallbackLocation(childPath); ok {
entry.KeyLocation = location
entry.ValueLocation = location
}
visit(entry.Value, childPath)
}
case document.Sequence:
for index, child := range node.Sequence {
visit(child, path.Index(index))
}
}
}
visit(located, Path{})
return located
}
func metadataFieldLocation(metadata Metadata, pointer string, origin Origin) (document.Location, bool) {
if location, ok := latestOriginDocumentLocation(origin); ok {
return location, true
}
for index := len(metadata.history[pointer]) - 1; index >= 0; index-- {
change := metadata.history[pointer][index]
if location, ok := latestOriginDocumentLocation(change.Origin); ok {
return location, true
}
for prior := len(change.Previous) - 1; prior >= 0; prior-- {
if location, ok := latestOriginDocumentLocation(change.Previous[prior]); ok {
return location, true
}
}
}
return document.Location{}, false
}
func latestOriginDocumentLocation(origin Origin) (document.Location, bool) {
if len(origin.Locations) == 0 {
return document.Location{}, false
}
location := origin.Locations[len(origin.Locations)-1]
return document.Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column}, true
}

Some files were not shown because too many files have changed in this diff Show More