From acd9e9a9fac337ddb2a2917468285eb155cf5c07 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 23 Oct 2026 19:49:12 +0000 Subject: [PATCH] Integrate collection policies across layered assembly --- docs/config.md | 46 ++++++---- docs/roadmap/implementation.md | 8 +- .../testdata/prompt-catalog/config.yml | 4 + .../testdata/prompt-catalog/invalid.yml | 5 + facade_sequence_test.go | 44 ++++++++- facade_source_version_mode_test.go | 6 +- .../engine/destination_accounting_test.go | 10 +- .../engine/destination_integration_test.go | 2 +- internal/engine/directives.go | 5 +- internal/engine/imports.go | 6 +- internal/engine/load.go | 6 +- internal/engine/load_input.go | 39 +++++++- internal/engine/load_request.go | 91 +++++++++++-------- internal/engine/load_request_test.go | 82 ++++++++++++++++- internal/engine/sequence_registration_test.go | 8 +- 15 files changed, 269 insertions(+), 93 deletions(-) diff --git a/docs/config.md b/docs/config.md index 6dbf3b1..25c2662 100644 --- a/docs/config.md +++ b/docs/config.md @@ -130,16 +130,20 @@ nothing. Underlay groups are listed from lowest to highest priority. Repeating a file within one group remains an import/repetition error; the same reusable source may be selected again in another tier. -ConfigKit acquires and composes the main input first, constructs the schema, -and checks the optional application version from the main tier only. Underlay -roots cannot claim that version path (`underlay.version`); imported fragments -retain `import.version` protection. It then acquires and composes underlay -groups in order, recursively combines the groups, and applies the main tier, -one selected overlay, and explicit CLI overrides. Scalar, sequence, and null -definitions replace lower values between tiers; mappings combine recursively. -Syntax and strict-composition errors inside a tier cannot be masked by a -higher-priority tier. Defaults, typed decoding, resolution, secret lookup, -validation, and publication run once on the final assembled input. +ConfigKit acquires the main input first and constructs the schema. It then +acquires and strictly composes underlay groups from lowest to highest priority, +passing each group's accumulated lower collection policy into the next group. +Each group is applied once as a precedence tier. The main tier is then composed +with the accumulated underlay policy, and the optional application version is +checked against that composed main tier only. Underlay roots cannot claim that +version path (`underlay.version`); imported fragments retain `import.version` +protection. ConfigKit then applies the selected overlay and explicit CLI +overrides. Scalar, sequence, and null definitions replace lower values between +tiers unless a collection policy defines the sequence operation; mappings +combine recursively. Syntax and strict-composition errors inside a tier cannot +be masked by a higher-priority tier. Defaults, typed decoding, resolution, +secret lookup, validation, and publication run once on the final assembled +input. An active layered overlay requires `LayeredOptions.OverlayFilesystem` explicitly, independent of the main source's namespace. Its name and root-relative lookup @@ -149,7 +153,10 @@ Ordinary `Load` continues to infer its selected-overlay context from its source. Source IDs follow acquisition order—main and its depth-first imports, then each underlay tier's inputs and depth-first imports, then overlay and CLI—even though -history follows precedence order. Source `Layer` describes precedence while +history follows precedence order. Default main-version checking happens after +underlay composition because the main tier inherits the accumulated collection +policy. Enabled per-source version checks retain acquisition order and source +local headers. Source `Layer` describes precedence while `Kind` describes acquisition. Underlay records use `LayerUnderlay` and a one-based `UnderlayIndex` giving their original tier position; all other records have index zero. Explicit empty groups and `NoSource` consume no source ID. @@ -824,8 +831,8 @@ headers are consumed before strict composition and cannot supply a registered record identity. Final typed decoding and application validation still run on the completed configuration. -Ordinary sources and selected overlays may declare collection behavior under -`configkit.sequences` at the document root. Each entry has a required literal +Ordinary sources, layered inputs, and selected overlays may declare collection +behavior under `configkit.sequences` at the document root. Each entry has a required literal `destination` and may specify `key` as one string or a nonempty list of strings, plus a `mode` of `replace`, `append`, or `upsert`. These controls are removed before application decoding. A declaration destination is relative to that @@ -841,11 +848,14 @@ replaces a complete matched record at its existing position and appends new identities in incoming order; it does not merge record fields. Declarations govern ordinary strict source composition, including loads with -imports disabled, and selected-overlay precedence. Overlay declarations apply -after the base source graph is composed and must agree with retained collection -identity and any application registration. They do not change layered -precedence, inheritance, or CLI behavior. A root `configkit.imports` -declaration still requires `Options.Imports`; `NoSource` has no declaration scope. +imports disabled, and each strict scope in layered loading. Layered groups and +main apply their resolved sequence operation between precedence tiers; a higher +strict scope inherits the lower identity and explicit mode before peer +composition. Overlay declarations apply after the base source graph is composed +and must agree with retained collection identity and any application +registration. These policies do not change inheritance or CLI behavior. A root +`configkit.imports` declaration still requires `Options.Imports`; `NoSource` has +no declaration scope. An empty mapping contributes no child definitions. Two empty mappings compose, and an empty mapping neither clears nor conflicts with another mapping's children. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7640fdb..a9e75de 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Sequence Identity And Combination Implementation Plan -Status: implementation in progress; Stages 1–14 complete. +Status: implementation in progress; Stages 1–15 complete. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. This plan starts at commit @@ -556,6 +556,12 @@ Do not absorb unfinished work from Stages 5–14 into this stage. Validation: layered/grouped, source-version, dependency scheduling, destination, and metadata tests plus the everyday check. +**Completion:** Implemented lower collection-policy propagation across underlay +groups, main composition, and layered overlays. Verified policy-only lower +tiers, inherited identity and explicit mode, explicit replacement, source +version behavior, sequence history, and consumer examples. Validation: +`go test ./internal/engine -count=1` and `./scripts/check.sh`. + ## Stage 16 — Verify Replacement And Nested Policy Ownership End To End Scope: the consequential lifetime cases across the now-active source assembly. diff --git a/examples/consumers/testdata/prompt-catalog/config.yml b/examples/consumers/testdata/prompt-catalog/config.yml index e59e761..c713ef2 100644 --- a/examples/consumers/testdata/prompt-catalog/config.yml +++ b/examples/consumers/testdata/prompt-catalog/config.yml @@ -1,4 +1,8 @@ configkit: + sequences: + - destination: [prompts] + key: id + mode: replace imports: - dir: prompts exclude: [generated-ignored.yml] diff --git a/examples/consumers/testdata/prompt-catalog/invalid.yml b/examples/consumers/testdata/prompt-catalog/invalid.yml index a916db6..49ed78d 100644 --- a/examples/consumers/testdata/prompt-catalog/invalid.yml +++ b/examples/consumers/testdata/prompt-catalog/invalid.yml @@ -1,3 +1,8 @@ +configkit: + sequences: + - destination: [prompts] + key: id + mode: replace prompts: - id: invalid name: Invalid prompt diff --git a/facade_sequence_test.go b/facade_sequence_test.go index f98bccc..01a18d5 100644 --- a/facade_sequence_test.go +++ b/facade_sequence_test.go @@ -107,7 +107,13 @@ func TestFacadeRegisteredReplacementExplainsRemovedDescendants(t *testing.T) { {"removed field", []any{map[string]any{"id": "main"}}, configkit.Keys("records").Index(0).Key("text")}, } { t.Run(test.name, func(t *testing.T) { - result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Mapping("main", map[string]any{"records": test.upper})}, configkit.LayeredOptions[map[string]any]{ + mainYAML := "configkit:\n sequences:\n - destination: [records]\n key: id\n mode: replace\n" + if len(test.upper) == 0 { + mainYAML += "records: []\n" + } else { + mainYAML += "records: [{id: main}]\n" + } + result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte(mainYAML))}, configkit.LayeredOptions[map[string]any]{ Definition: configkit.Definition[map[string]any]{SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("records"), IdentityField: "id"}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("lower", map[string]any{"records": []any{map[string]any{"id": "one", "text": "old"}, map[string]any{"id": "two"}}})}}}}, }) @@ -240,8 +246,8 @@ func TestFacadeSequencePlacementControlsAcrossLoadEntryPoints(t *testing.T) { if err != nil { t.Fatalf("registered sequence precedence load: %v", err) } - if got := precedence.Config["entries"].([]any); len(got) != 1 || got[0].(map[string]any)["id"] != "main" { - t.Fatalf("higher-priority list did not replace the lower sequence: %#v", precedence.Config) + if got := precedence.Config["entries"].([]any); len(got) != 2 || got[0].(map[string]any)["id"] != "lower" || got[1].(map[string]any)["id"] != "main" { + t.Fatalf("registered upsert did not retain the lower sequence: %#v", precedence.Config) } for _, test := range []struct { @@ -506,6 +512,31 @@ func TestFacadeSourceInfoPlacementCopiesAndZeroValues(t *testing.T) { } } +func TestLayeredSequenceUpsertHistoryRetainsLowerTierRecords(t *testing.T) { + definition := configkit.Definition[map[string]any]{ + SequenceCompositions: []configkit.SequenceComposition{{Path: configkit.Keys("entries"), IdentityField: "id"}}, + } + result, err := configkit.LoadLayered[map[string]any](configkit.Input{Source: configkit.Bytes("main", []byte("entries: [{id: main}]\n"))}, configkit.LayeredOptions[map[string]any]{ + Definition: definition, + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{ + Source: configkit.Bytes("lower", []byte("entries: [{id: lower}]\n")), + }}}}, + }) + if err != nil { + t.Fatal(err) + } + for index, id := range []string{"lower", "main"} { + path := configkit.Keys("entries").Index(index).Key("id") + explanation, ok := result.Explain(path) + if !ok || !explanation.Effective.Present || explanation.Effective.YAML != id+"\n" || len(explanation.History) == 0 { + t.Fatalf("tier history at %s = %#v, %v", path.String(), explanation, ok) + } + if explanation.History[len(explanation.History)-1].After.YAML != id+"\n" { + t.Fatalf("tier history at %s does not retain final record: %#v", path.String(), explanation.History) + } + } +} + func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *testing.T) { type record struct { ID string `yaml:"id"` @@ -523,7 +554,12 @@ func TestFacadeSequenceReplacementKeepsTypedAndHistoricalEvidenceAligned(t *test directory := t.TempDir() mainPath := filepath.Join(directory, "main.yml") underlayPath := filepath.Join(directory, "underlay.yml") - if err := os.WriteFile(mainPath, []byte(`entries: + if err := os.WriteFile(mainPath, []byte(`configkit: + sequences: + - destination: [entries] + key: id + mode: replace +entries: - id: current count: 0 enabled: false diff --git a/facade_source_version_mode_test.go b/facade_source_version_mode_test.go index 0697538..39b981c 100644 --- a/facade_source_version_mode_test.go +++ b/facade_source_version_mode_test.go @@ -104,11 +104,7 @@ func TestFacadeSourceVersionOwnershipAcceptsContextualIndexes(t *testing.T) { if err != nil || result == nil { t.Fatalf("grouped=%t path=%#v: %v", grouped, path, err) } - want := 2 - if grouped { - want = 1 - } - if len(result.Config["records"].([]any)) != want { + if len(result.Config["records"].([]any)) != 2 { t.Fatal("version path changed composition") } } diff --git a/internal/engine/destination_accounting_test.go b/internal/engine/destination_accounting_test.go index e541faa..4ed7b28 100644 --- a/internal/engine/destination_accounting_test.go +++ b/internal/engine/destination_accounting_test.go @@ -34,7 +34,7 @@ func TestDestinationPlacementCountsAgainstSharedNodeBudget(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() accounting := &importAccounting{nodes: limits.nodes - perInput + test.delta} - composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ + composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{ source: empty, destination: destination, }}}, 1, nil, accounting) if test.wantError { @@ -62,7 +62,7 @@ func TestMultipleMountedInputsSharePlacementCharges(t *testing.T) { perInput := countInputNodes(&document.Node{Kind: document.Mapping}) + destinationWrapperNodeCount(Keys("collection", "one")) charge := len(inputs) * perInput accounting := &importAccounting{nodes: limits.nodes - charge} - composition, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting) + composition, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: inputs}, 1, nil, accounting) if len(diagnostics) != 0 || composition == nil || accounting.nodes != limits.nodes || composition.inputNodes != charge { t.Fatalf("shared placement accounting = %#v, %#v, accounting %#v; want charge %d", composition, diagnostics, accounting, charge) } @@ -107,7 +107,7 @@ id: parent } { t.Run(test.name, func(t *testing.T) { accounting := &importAccounting{nodes: limits.nodes - wantNodes + test.delta} - composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ + composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{ source: File(mainPath), imports: true, destination: destination, placementMode: placementModeSequenceItem, }, { source: Mapping("other", map[string]any{"id": "other"}), destination: otherDestination, placementMode: placementModeSequenceItem, @@ -160,7 +160,7 @@ func TestImportedEmptyBodiesChargeEachDestinationWrapper(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() accounting := &importAccounting{nodes: limits.nodes - rawNodes - placementNodes + test.delta} - composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{ + composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{ source: mainSource, imports: true, destination: destination, }}}, 1, nil, accounting) if test.wantError { @@ -387,7 +387,7 @@ func TestNoSourceAndRootPlacementAddNoWrapperCharges(t *testing.T) { t.Parallel() accounting := &importAccounting{} - composition, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{ + composition, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{ {source: NoSource(), destination: Keys("absent")}, {source: Mapping("root", map[string]any{})}, }}, 1, nil, accounting) diff --git a/internal/engine/destination_integration_test.go b/internal/engine/destination_integration_test.go index b1c5f11..b56a5f6 100644 --- a/internal/engine/destination_integration_test.go +++ b/internal/engine/destination_integration_test.go @@ -224,7 +224,7 @@ func TestLayeredItemVersionProtectionTracksAssignedPeerIndexes(t *testing.T) { t.Fatalf("unrelated appended peer rejected: %v", err) } entries := layeredMapValue(t, testLoaded(loaded).Config, "catalog", "entries").([]any) - if len(entries) != 1 || entries[0].(map[string]any)["id"] != "main" { + if len(entries) != 3 || entries[0].(map[string]any)["id"] != "first" || entries[1].(map[string]any)["id"] != "peer" || entries[2].(map[string]any)["id"] != "main" { t.Fatalf("main item projection = %#v", entries) } }) diff --git a/internal/engine/directives.go b/internal/engine/directives.go index effd1f3..d429a11 100644 --- a/internal/engine/directives.go +++ b/internal/engine/directives.go @@ -15,6 +15,7 @@ const ( rootDirectiveContextImportsOnly rootDirectiveContext = iota + 1 rootDirectiveContextSource rootDirectiveContextOverlay + rootDirectiveContextLayered ) type rootDirectiveExtraction struct { @@ -78,7 +79,7 @@ func parseRootDirectiveBlock(node *document.Node, directivePath Path, context ro diagnostics = append(diagnostics, nodeImportDiagnostic(CodeOverlayDirective, fieldPath, "overlays cannot contain imports", entry.Value)) continue } - if context != rootDirectiveContextImportsOnly && context != rootDirectiveContextSource { + if context != rootDirectiveContextImportsOnly && context != rootDirectiveContextSource && context != rootDirectiveContextLayered { diagnostics = append(diagnostics, nodeImportDiagnostic(CodeImportInvalid, fieldPath, "directive extraction context is invalid", entry.Value)) continue } @@ -91,7 +92,7 @@ func parseRootDirectiveBlock(node *document.Node, directivePath Path, context ro diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "sequence declarations are not enabled in this extraction context", entry.Value)) continue } - if context != rootDirectiveContextSource && context != rootDirectiveContextOverlay { + if context != rootDirectiveContextSource && context != rootDirectiveContextOverlay && context != rootDirectiveContextLayered { diagnostics = append(diagnostics, sequenceDeclarationDiagnostic(fieldPath, "directive extraction context is invalid", entry.Value)) continue } diff --git a/internal/engine/imports.go b/internal/engine/imports.go index b51adcc..eb0a4a3 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -286,7 +286,7 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition), sources: state.sources, versionState: versionState, accounting: accounting, } - composed, diagnostics := composeAcquiredInputWithPolicy(input, policy) + composed, diagnostics := composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy) return composed, cloneSourceInfos(state.sources), diagnostics } @@ -294,10 +294,6 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem // acquisition scopes to the established strict-composition behavior. It uses // the already acquired documents and graph; source handles are closed before // this handoff. -func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { - return composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy) -} - func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { return input.composition, nil diff --git a/internal/engine/load.go b/internal/engine/load.go index 9d96d25..bafa60e 100644 --- a/internal/engine/load.go +++ b/internal/engine/load.go @@ -169,13 +169,15 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode} mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults) mainOptions.sequencePolicy = request.sequencePolicy - mainOptions.deferAssembly = request.ordinary + mainOptions.deferAssembly = true mainOptions.SourceVersionMode = request.sourceVersionMode mainOptions.Definition.Version = request.protectionVersion() mainOptions.Overrides = request.overrides if request.ordinary { mainOptions.Overlay = request.overlay mainOptions.directiveContext = rootDirectiveContextSource + } else { + mainOptions.directiveContext = rootDirectiveContextLayered } var schema *schemaDescriptor @@ -216,7 +218,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo } input.document = input.composition.root } - if request.sourceVersionMode == sourceVersionModeDefault { + if request.sourceVersionMode == sourceVersionModeDefault && request.ordinary { versionDiagnostics := checkVersion(request.definition.Version, input.document) accumulator.add(versionDiagnostics...) if hasErrors(versionDiagnostics) { diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index 74b8d16..e87a2bf 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -129,7 +129,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d } if !options.Imports && parsed.document != nil { var directiveDiagnostics []Diagnostic - if directiveContext == rootDirectiveContextSource { + if directiveContext == rootDirectiveContextSource || directiveContext == rootDirectiveContextLayered { directiveDiagnostics = disabledRootImportDiagnostics(parsed.document) } else { directiveDiagnostics = disabledDirectiveDiagnostics(parsed.document, Path{}) @@ -159,6 +159,33 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d if role == sourceVersionMain && versionState != nil { versionState.mainProcessing = &processed } + } else if directiveContext == rootDirectiveContextLayered { + extracted, failures := extractRootDirectives(parsed.document, rootDirectiveContextLayered) + if len(failures) != 0 { + return input, failures + } + processed := sourceVersionProcessingResult{ + payload: extracted.body, imports: extracted.imports, sequences: extracted.sequences, + directiveContext: rootDirectiveContextLayered, + } + if versionState != nil && versionState.mode != sourceVersionModeDefault { + var sourceID string + if parsed.source != nil { + sourceID = parsed.source.ID + } + versionProcessed, versionFailures := versionState.process(extracted.body, role, destination, sourceID, options.placementMode) + if len(versionFailures) != 0 { + return input, versionFailures + } + versionProcessed.imports = extracted.imports + versionProcessed.sequences = append(cloneSequenceDeclarations(extracted.sequences), versionProcessed.sequences...) + versionProcessed.directiveContext = rootDirectiveContextLayered + processed = versionProcessed + if role == sourceVersionMain { + versionState.mainProcessing = &processed + } + } + versionProcessing = &processed } else if versionState != nil && versionState.mode != sourceVersionModeDefault { var sourceID string if parsed.source != nil { @@ -302,9 +329,13 @@ func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic) } func compositionForInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { + return compositionForInputWithPolicies(input, collectionPolicyScope{}, policy) +} + +func compositionForInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { if input.composition != nil { if input.composition.root == nil && len(input.composition.acquisition) != 0 { - return composeAcquiredInputWithPolicy(input, policy) + return composeAcquiredInputWithPolicies(input, lower, policy) } return input.composition, nil } @@ -318,12 +349,12 @@ func compositionForInputWithPolicy(input assembledInput, policy sequencePolicy) acquired.inputNodes = countInputNodes(input.parsed.document) } input.composition = acquired - return composeAcquiredInputWithPolicy(input, policy) + return composeAcquiredInputWithPolicies(input, lower, policy) } if input.document == nil { return &composedDocument{root: &document.Node{Kind: document.Mapping}, origins: make(map[string]Origin)}, nil } - composition, diagnostics := composeDocumentsWithPolicy(policy, input.document) + composition, diagnostics := composeDocumentsWithCollectionPolicy(policy, lower, nil, input.document) if len(diagnostics) != 0 { return nil, diagnostics } diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index c70af75..d49dbb2 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -92,7 +92,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi } func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) { - groups := make([]*composedDocument, 0, len(request.underlays)) + var assembly *assemblyAccumulator for index, group := range request.underlays { acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy) mergeResourceContexts(&input.contexts, acquired.contexts) @@ -100,35 +100,71 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a input.sources = append(input.sources, acquired.sources...) return input, diagnostics } - composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, collectionPolicyScope{}) + var lower collectionPolicyScope + if assembly != nil && assembly.current != nil { + lower = assembly.current.collectionPolicies + } + composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, lower) input.sources = append(input.sources, acquired.sources...) if len(diagnostics) != 0 { return input, diagnostics } if composition != nil { - groups = append(groups, composition) + if assembly == nil { + assembly = newAssemblyAccumulator(composition) + } else { + var declarations []sequenceDeclarationSpec + for _, scope := range composition.acquisition { + declarations = append(declarations, acquiredSequenceDeclarations(scope)...) + } + if diagnostics := assembly.applyOverlayWithCollectionPolicy( + composition.root, composition.origins, composition.applications, LayerUnderlay, + request.sequencePolicy, declarations, + ); len(diagnostics) != 0 { + return input, diagnostics + } + } } } - var assembly *assemblyAccumulator - for _, group := range groups { - if assembly == nil { - assembly = newAssemblyAccumulator(group) - continue + mainPresent := input.document != nil || input.composition != nil || len(input.acquisition) != 0 + var mainComposition *composedDocument + if mainPresent { + var lower collectionPolicyScope + if assembly != nil && assembly.current != nil { + lower = assembly.current.collectionPolicies } - if diagnostics := assembly.applyOverlay(group.root, group.origins, group.applications, LayerUnderlay); len(diagnostics) != 0 { - return input, diagnostics - } - } - if input.document != nil { - mainComposition, diagnostics := compositionForInputWithPolicy(input, request.sequencePolicy) + var diagnostics []Diagnostic + mainComposition, diagnostics = compositionForInputWithPolicies(input, lower, request.sequencePolicy) if len(diagnostics) != 0 { return input, diagnostics } + } + if request.sourceVersionMode == sourceVersionModeDefault { + var mainRoot *document.Node + if mainComposition != nil { + mainRoot = mainComposition.root + } else if input.document != nil { + mainRoot = input.document + } + if diagnostics := checkVersion(request.definition.Version, mainRoot); hasErrors(diagnostics) { + return input, diagnostics + } + } + if mainComposition != nil { if assembly == nil { assembly = newAssemblyAccumulator(mainComposition) - } else if diagnostics := assembly.applyOverlay(mainComposition.root, mainComposition.origins, mainComposition.applications, LayerBase); len(diagnostics) != 0 { - return input, diagnostics + } else { + var declarations []sequenceDeclarationSpec + for _, scope := range mainComposition.acquisition { + declarations = append(declarations, acquiredSequenceDeclarations(scope)...) + } + if diagnostics := assembly.applyOverlayWithCollectionPolicy( + mainComposition.root, mainComposition.origins, mainComposition.applications, LayerBase, + request.sequencePolicy, declarations, + ); len(diagnostics) != 0 { + return input, diagnostics + } } } if assembly == nil && (activeOverlay(request.overlay) || len(request.overrides) != 0) { @@ -146,10 +182,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a selection := Source{kind: sourceBytes} selected := parsedSource{document: &document.Node{Kind: document.Mapping}, source: &SourceInfo{}} var diagnostics []Diagnostic - var overlayPolicy []sequencePolicy - if request.ordinary { - overlayPolicy = []sequencePolicy{request.sequencePolicy} - } + overlayPolicy := []sequencePolicy{request.sequencePolicy} input.sources, diagnostics = loadSelectedOverlayIntoWithVersion(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.protectionVersion(), defaultImportLimits(), input.versionState, overlayPolicy...) if len(diagnostics) != 0 { return input, diagnostics @@ -172,23 +205,6 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a return input, nil } -func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { - return acquireUnderlayGroupWithVersion(group, index, rule, accounting, nil) -} - -func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { - return acquireUnderlayGroupWithPolicyVersion(group, index, rule, accounting, versionState, sequencePolicy{}) -} - -func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { - acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, versionState, policy) - if len(diagnostics) != 0 { - return nil, acquired.sources, acquired.contexts, diagnostics - } - composition, diagnostics := composeUnderlayGroup(acquired, rule, policy, collectionPolicyScope{}) - return composition, acquired.sources, acquired.contexts, diagnostics -} - func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) { initialBytes, initialNodes := accounting.bytes, accounting.nodes visited := make(map[resourceIdentity]importRoute) @@ -208,6 +224,7 @@ func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, a Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}, destination: contribution.destination.clone(), placementMode: contribution.placementMode, sequencePolicy: policy, deferAssembly: true, + directiveContext: rootDirectiveContextLayered, importDefaults: func() importDefaults { if versionState == nil { return importDefaults{} diff --git a/internal/engine/load_request_test.go b/internal/engine/load_request_test.go index 8cfbe0b..6e2f1eb 100644 --- a/internal/engine/load_request_test.go +++ b/internal/engine/load_request_test.go @@ -355,6 +355,69 @@ func TestGroupedLoadAbsentMainAndVersionProtection(t *testing.T) { } } +func TestLayeredCollectionPolicyFlowsBetweenTiers(t *testing.T) { + t.Parallel() + + policyOnly := Bytes("lower-policy", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\n")) + duplicate := func(name string) loadInput { + return loadInput{source: Bytes(name, []byte("items: [{id: repeated}]\n"))} + } + request := loadRequest[map[string]any]{ + main: loadInput{source: NoSource()}, + underlays: []loadGroup{ + {inputs: []loadInput{{source: policyOnly}}}, + {inputs: []loadInput{duplicate("first-peer"), duplicate("second-peer")}}, + }, + } + if _, err := testGroupedLoad(request); err != nil && groupedErrorCode(t, err) != CodeComposeConflict { + loadErr, _ := err.(*LoadError) + t.Fatalf("higher strict group code = %s, want %s: %#v", groupedErrorCode(t, err), CodeComposeConflict, loadErr.Diagnostics()) + } else if err == nil { + t.Fatal("higher strict group accepted duplicate inherited identities") + } + + request = loadRequest[map[string]any]{ + main: loadInput{source: NoSource()}, + underlays: []loadGroup{ + {inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: repeated}]\n"))}}}, + {inputs: []loadInput{duplicate("higher")}}, + }, + } + if _, err := testGroupedLoad(request); err == nil || groupedErrorCode(t, err) != CodeSequenceConflict { + t.Fatalf("higher tier did not inherit explicit append mode: %v", err) + } + + request = loadRequest[map[string]any]{ + main: loadInput{source: Bytes("main", []byte("configkit:\n sequences:\n - destination: [items]\n key: name\n mode: replace\nitems: [{name: higher}]\n"))}, + underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: lower}]\n"))}}}}, + } + result, err := testGroupedLoad(request) + if err != nil || !reflect.DeepEqual(result.Config["items"], []any{map[string]any{"name": "higher"}}) { + var diagnostics []Diagnostic + if loadErr, ok := err.(*LoadError); ok { + diagnostics = loadErr.Diagnostics() + } + t.Fatalf("replacement did not retire lower identity: %#v, %v (%#v)", result, err, diagnostics) + } + + filesystem := NewFilesystem("overlay", fstest.MapFS{ + "overlays/testing.yml": &fstest.MapFile{Data: []byte("configkit:\n sequences:\n - destination: [items]\n key: id\n mode: append\nitems: [{id: overlay}]\n")}, + }) + definition := Definition[map[string]any]{sequenceRegistrations: []sequenceRegistration{{path: Keys("items"), identityField: "id"}}} + request = loadRequest[map[string]any]{ + main: loadInput{source: Bytes("main", []byte("items: [{id: main}]\n"))}, + underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("items: [{id: lower}]\n"))}}}}, + definition: definition, sequencePolicy: mustCompileSequencePolicy(t, definition.sequenceRegistrations), + overlay: "testing", overlayFilesystem: &FilesystemContext{Root: ".", BaseDir: ".", FS: &filesystem}, + } + result, err = testGroupedLoad(request) + if err != nil || !reflect.DeepEqual(result.Config["items"], []any{ + map[string]any{"id": "lower"}, map[string]any{"id": "main"}, map[string]any{"id": "overlay"}, + }) { + t.Fatalf("layered selected overlay sequence policy = %#v, %v", result, err) + } +} + func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) { main := loadInput{source: Bytes("main", []byte("port: 5\n"))} request := loadRequest[groupedConfig]{ @@ -393,18 +456,27 @@ func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) { accounting := &importAccounting{sources: maxImportSources - 1} group := loadGroup{inputs: []loadInput{{source: Mapping("first", map[string]any{})}, {source: Mapping("second", map[string]any{})}}} - _, _, _, diagnostics := acquireUnderlayGroup(group, 1, nil, accounting) + _, _, _, diagnostics := testAcquireUnderlayGroup(group, 1, nil, accounting) if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded { t.Fatalf("shared source budget = %#v", diagnostics) } accounting = &importAccounting{bytes: maxTotalBytes - 2} group.inputs = []loadInput{{source: Mapping("first", map[string]any{"port": "one"})}} - _, _, _, diagnostics = acquireUnderlayGroup(group, 1, nil, accounting) + _, _, _, diagnostics = testAcquireUnderlayGroup(group, 1, nil, accounting) if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded { t.Fatalf("shared byte budget = %#v", diagnostics) } } +func testAcquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { + acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, nil, sequencePolicy{}) + if len(diagnostics) != 0 { + return nil, acquired.sources, acquired.contexts, diagnostics + } + composition, diagnostics := composeUnderlayGroup(acquired, rule, sequencePolicy{}, collectionPolicyScope{}) + return composition, acquired.sources, acquired.contexts, diagnostics +} + func TestGroupedLoadInvalidIntermediateTypeCanBeReplaced(t *testing.T) { request := loadRequest[groupedConfig]{ main: loadInput{source: Bytes("main", []byte("port: 9\n"))}, @@ -446,11 +518,11 @@ func TestGroupedAccountingChargesMixedOccurrencesOnce(t *testing.T) { "child-two.yml": &fstest.MapFile{Data: []byte(child)}, }) accounting := &importAccounting{} - first, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("one.yml"), imports: true}}}, 1, nil, accounting) + first, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("one.yml"), imports: true}}}, 1, nil, accounting) if len(diagnostics) != 0 || first == nil || len(sources) != 2 || accounting.sources != 2 || accounting.bytes != int64(len(one)+len(child)) { t.Fatalf("first acquisition = %#v, %#v, %#v, accounting %#v", first, sources, diagnostics, accounting) } - second, sources, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("two.yml"), imports: true}}}, 2, nil, accounting) + second, sources, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("two.yml"), imports: true}}}, 2, nil, accounting) if len(diagnostics) != 0 || second == nil || len(sources) != 2 || sources[0].ID != "source-3" || sources[1].ID != "source-4" || accounting.sources != 4 || accounting.bytes != int64(len(one)+len(two)+2*len(child)) { t.Fatalf("second acquisition = %#v, %#v, %#v, accounting %#v", second, sources, diagnostics, accounting) } @@ -477,7 +549,7 @@ func TestGroupedGlobEntryBudgetIsShared(t *testing.T) { "parts/b.yml": &fstest.MapFile{Data: []byte("right: two\n")}, }) accounting := &importAccounting{entries: maxGlobEntries - 1} - _, _, _, diagnostics := acquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}, 1, nil, accounting) + _, _, _, diagnostics := testAcquireUnderlayGroup(loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}, 1, nil, accounting) if len(diagnostics) == 0 || diagnostics[0].Code != CodeImportLimitExceeded || accounting.entries <= maxGlobEntries { t.Fatalf("shared glob budget = %#v, accounting %#v", diagnostics, accounting) } diff --git a/internal/engine/sequence_registration_test.go b/internal/engine/sequence_registration_test.go index 7bac837..f80e4b7 100644 --- a/internal/engine/sequence_registration_test.go +++ b/internal/engine/sequence_registration_test.go @@ -320,7 +320,7 @@ func TestRegisteredSequencePolicyAcrossLoadingLifecycle(t *testing.T) { } }) - t.Run("strict underlay group appends while separate tiers reset identities", func(t *testing.T) { + t.Run("registered sequence policy appends across strict groups and tiers", func(t *testing.T) { request := loadRequest[config]{ main: loadInput{source: Bytes("main", []byte("items: [{id: shared}]\n"))}, underlays: []loadGroup{{inputs: []loadInput{ @@ -330,7 +330,7 @@ func TestRegisteredSequencePolicyAcrossLoadingLifecycle(t *testing.T) { definition: definition, sequencePolicy: mustCompileSequencePolicy(t, definition.sequenceRegistrations), } result, err := testGroupedLoad(request) - if err != nil || len(result.Config.Items) != 1 || result.Config.Items[0].ID != "shared" { + if err != nil || len(result.Config.Items) != 3 || result.Config.Items[0].ID != "lower" || result.Config.Items[1].ID != "other" || result.Config.Items[2].ID != "shared" { t.Fatalf("grouped load = %#v, %v", result, err) } @@ -344,7 +344,7 @@ func TestRegisteredSequencePolicyAcrossLoadingLifecycle(t *testing.T) { } }) - t.Run("empty registered contribution clears lower sequence", func(t *testing.T) { + t.Run("empty registered contribution follows the accumulated sequence mode", func(t *testing.T) { result, err := testLoad[config](Bytes("main", []byte("items: []\n")), registered) if err != nil || len(result.Config.Items) != 0 { t.Fatalf("empty sequence load = %#v, %v", result, err) @@ -355,7 +355,7 @@ func TestRegisteredSequencePolicyAcrossLoadingLifecycle(t *testing.T) { definition: definition, sequencePolicy: mustCompileSequencePolicy(t, definition.sequenceRegistrations), } result, err = testGroupedLoad(request) - if err != nil || len(result.Config.Items) != 0 { + if err != nil || len(result.Config.Items) != 1 || result.Config.Items[0].ID != "lower" { t.Fatalf("empty higher tier = %#v, %v", result, err) } })