eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Project incoming source destinations through tier operations

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-24 00:05:24 +00:00
parent a035df9845
commit f7624ae92a
6 changed files with 180 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/roadmap/remediation.md
View File

@@ -138,7 +138,14 @@ agree with structural assembly before real decoding reconciliation.
## 4. Reproject Nested Source Destinations After Cross-Tier Item Movement — P2
Status: [ ] Open.
Status: [x] Complete.
Completion: layered assembly projects incoming source destinations and version
classification evidence with the same item assignments used for values/history.
Earlier contributions retain their historical coordinates, physical identities
and contexts. Main/underlay regressions cover reordered upserts, append, replace,
nested helpers, and source-local version evidence. Verified focused destination,
origin and source-version tests, focused race coverage, and `./scripts/check.sh`.
Owners: [`projectAcquiredDestinations`](../../internal/engine/imports.go) and the
sequence-operation handoff in [`assembleLayeredInput`](../../internal/engine/load_request.go).

16
internal/engine/load_request.go
View File

@@ -93,7 +93,10 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
var assembly *assemblyAccumulator
mainSourceCount := len(input.sources)
mainEvidence := input.versionState.evidenceSince(0)
for index, group := range request.underlays {
evidenceStart := len(input.versionState.evidenceSince(0))
acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
mergeResourceContexts(&input.contexts, acquired.contexts)
if len(diagnostics) != 0 {
@@ -105,6 +108,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
lower = assembly.current.collectionPolicies
}
composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, lower)
sourceStart := len(input.sources)
input.sources = append(input.sources, acquired.sources...)
if len(diagnostics) != 0 {
return input, diagnostics
@@ -117,12 +121,14 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
for _, scope := range composition.acquisition {
declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
}
if diagnostics := assembly.applyOverlayWithCollectionPolicy(
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
composition.root, composition.origins, composition.collectionPolicies, composition.applications, LayerUnderlay,
request.sequencePolicy, declarations,
); len(diagnostics) != 0 {
)
if len(diagnostics) != 0 {
return input, diagnostics
}
projectTierSourceDestinations(input.sources[sourceStart:], input.versionState.evidenceSince(evidenceStart), operations)
}
}
}
@@ -159,12 +165,14 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
for _, scope := range mainComposition.acquisition {
declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
}
if diagnostics := assembly.applyOverlayWithCollectionPolicy(
operations, diagnostics := assembly.applyOverlayWithCollectionProjection(
mainComposition.root, mainComposition.origins, mainComposition.collectionPolicies, mainComposition.applications, LayerBase,
request.sequencePolicy, declarations,
); len(diagnostics) != 0 {
)
if len(diagnostics) != 0 {
return input, diagnostics
}
projectTierSourceDestinations(input.sources[:mainSourceCount], mainEvidence, operations)
}
}
if assembly == nil && activeOverlay(request.overlay) {

21
internal/engine/overlay.go
View File

@@ -65,13 +65,28 @@ func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy(
sequencePolicy sequencePolicy,
declarations []sequenceDeclarationSpec,
) []Diagnostic {
_, diagnostics := a.applyOverlayWithCollectionProjection(overlay, incomingOrigins, incomingPolicies, applications, layer, sequencePolicy, declarations)
return diagnostics
}
// applyOverlayWithCollectionProjection also returns the incoming tier's item
// assignments so acquisition evidence can follow the same operation as values.
func (a *assemblyAccumulator) applyOverlayWithCollectionProjection(
overlay *document.Node,
incomingOrigins map[string]Origin,
incomingPolicies collectionPolicyScope,
applications []historyApplication,
layer Layer,
sequencePolicy sequencePolicy,
declarations []sequenceDeclarationSpec,
) ([]sequenceHistoryContribution, []Diagnostic) {
if a == nil || a.current == nil || a.current.root == nil || a.current.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping {
return invalidOverlayDocument()
return nil, invalidOverlayDocument()
}
mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer))
sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, incomingPolicies, layer, sequencePolicy, declarations)
if len(diagnostics) != 0 {
return diagnostics
return nil, diagnostics
}
if len(applications) == 0 {
application := historyApplication{layer: layer, value: overlay, sequences: sequences}
@@ -83,7 +98,7 @@ func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy(
a.current.applications = append(a.current.applications, application)
}
}
return nil
return sequences, nil
}
func mergeOverlayMappingsWithCollectionPolicy(

102
internal/engine/sequence_destination_projection_test.go Normal file
View File

@@ -0,0 +1,102 @@
package engine
import (
"fmt"
"strings"
"testing"
"testing/fstest"
)
func TestLayeredSequenceProjectsNestedSourceDestinations(t *testing.T) {
t.Parallel()
type catalog struct {
Items []map[string]any `yaml:"items"`
}
for _, mode := range []string{"upsert", "append", "replace"} {
for _, underlay := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/underlay=%t", mode, underlay), func(t *testing.T) {
files := fstest.MapFS{}
for _, id := range []string{"a", "b", "c"} {
files[id+"/record.yml"] = &fstest.MapFile{Data: []byte("configkit: {imports: [{path: helper.yml}]}\nid: " + id + "\n")}
files[id+"/helper.yml"] = &fstest.MapFile{Data: []byte("value: " + id + "\n")}
}
files["main.yml"] = &fstest.MapFile{Data: []byte("configkit:\n sequences: [{destination: [items], key: id, mode: " + mode + "}]\n imports: [{glob: '*/record.yml', destination: [items], placement: sequence-item}]\n")}
lower := "items: [{id: b}, {id: a}, {id: keep}]\n"
want := map[string]int{"a": 1, "b": 0, "c": 3}
if mode == "append" {
lower = "items: [{id: keep}]\n"
want = map[string]int{"a": 1, "b": 2, "c": 3}
} else if mode == "replace" {
want = map[string]int{"a": 0, "b": 1, "c": 2}
}
incoming := loadInput{source: NewFilesystem("catalog", files).File("main.yml"), imports: true}
request := loadRequest[catalog]{main: incoming, underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(lower))}}}}}
if underlay {
request.main = loadInput{source: NoSource()}
request.underlays = append(request.underlays, loadGroup{inputs: []loadInput{incoming}})
}
loaded, err := testGroupedLoad(request)
if err != nil {
t.Fatal(err)
}
helpers := 0
for _, source := range loaded.Sources() {
if !strings.HasSuffix(source.Path, "/helper.yml") {
continue
}
helpers++
context, ok := loaded.SourceContext(source.ID)
if !ok {
t.Fatal("missing declaring context")
}
index, ok := want[context.BaseDir]
if !ok || source.Destination.String() != Keys("items").Index(index).String() {
t.Errorf("%s destination = %s, want item %d", context.BaseDir, source.Destination, index)
}
if loaded.Config.Items[index]["value"] != context.BaseDir {
t.Errorf("source destination identifies another record: %#v", loaded.Config.Items[index])
}
}
if helpers != 3 {
t.Fatalf("retained %d helpers, want 3", helpers)
}
})
}
}
}
func TestTierProjectionPreservesSourceLocalVersionEvidence(t *testing.T) {
t.Parallel()
local := Keys("version")
destination := Keys("items").Index(0).Key("details")
lower := &sourceVersionEvidence{sourceID: "lower", path: local, destination: destination.clone()}
higher := &sourceVersionEvidence{
sourceID: "higher", path: local, resolvedPath: local,
destination: destination.clone(), version: Version{Text: "accepted"},
projectedPaths: []Path{appendDestination(destination, local), destination.clone()},
}
sources := []SourceInfo{{ID: "higher", Path: "record/helper.yml", Destination: destination.clone()}}
state := &sourceVersionState{evidence: []*sourceVersionEvidence{lower, higher}}
projectTierSourceDestinations(sources, state.evidenceSince(1), []sequenceHistoryContribution{{
path: Keys("items"), mode: sequenceHistoryMapped,
items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 1}},
}})
want := Keys("items").Index(1).Key("details")
if sources[0].Destination.String() != want.String() || sources[0].Path != "record/helper.yml" || higher.destination.String() != want.String() {
t.Fatalf("projected source/evidence = %#v, %#v", sources, higher)
}
if lower.destination.String() != destination.String() || lower.projectedPaths != nil {
t.Fatal("incoming projection relabeled a lower historical contribution")
}
if higher.path.String() != local.String() || higher.resolvedPath.String() != local.String() {
t.Fatal("projection changed source-local version coordinates")
}
paths := higher.sensitivityPaths()
if len(paths) != 2 || paths[0].String() != want.Key("version").String() || paths[1].String() != want.String() {
t.Fatalf("projected sensitivity paths = %#v", paths)
}
check := sourceVersionCheckForEvidence(higher, newDiffSensitivity(map[string]bool{want.String(): true}, nil), false)
if !check.Redacted || check.Version.Text != "" || check.Path.String() != local.String() {
t.Fatalf("version classification did not use effective coordinates: %#v", check)
}
}

35
internal/engine/sequence_history_projection.go
View File

@@ -16,7 +16,7 @@ func projectTierHistoryApplications(applications []historyApplication, operation
projected := cloneHistoryApplications(applications)
for index, original := range applications {
application := &projected[index]
if path, found := projectSequenceHistoryPath(original.path, projections); found {
if path, found := projectSequenceOperationPath(original.path, projections); found {
application.path = path
}
for sequenceIndex, source := range original.sequences {
@@ -55,7 +55,7 @@ func projectTierHistoryApplications(applications []historyApplication, operation
return projected
}
func projectSequenceHistoryPath(path Path, projections map[string]sequenceHistoryContributionIndex) (Path, bool) {
func projectSequenceOperationPath(path Path, projections map[string]sequenceHistoryContributionIndex) (Path, bool) {
projected := path.clone()
for index, segment := range path.segments {
if segment.kind != pathSegmentIndex {
@@ -74,3 +74,34 @@ func projectSequenceHistoryPath(path Path, projections map[string]sequenceHistor
}
return projected, true
}
// Project only incoming sources. Lower/displaced sources retain the coordinates
// of their own historical contribution, rather than becoming incoming survivors.
func projectTierSourceDestinations(sources []SourceInfo, versions []*sourceVersionEvidence, operations []sequenceHistoryContribution) {
if len(operations) == 0 {
return
}
projections := indexSequenceHistoryContributions(operations)
for index := range sources {
if path, found := projectSequenceOperationPath(sources[index].Destination, projections); found {
sources[index].Destination = path
}
}
for _, evidence := range versions {
if evidence == nil {
continue
}
paths := evidence.sensitivityPaths()
projected := make([]Path, len(paths))
for index, path := range paths {
projected[index] = path.clone()
if destination, found := projectSequenceOperationPath(path, projections); found {
projected[index] = destination
}
}
evidence.projectedPaths = projected
if path, found := projectSequenceOperationPath(evidence.destination, projections); found {
evidence.destination = path
}
}
}

7
internal/engine/source_version.go
View File

@@ -53,6 +53,13 @@ type sourceVersionState struct {
diagnostics []Diagnostic
}
func (state *sourceVersionState) evidenceSince(start int) []*sourceVersionEvidence {
if state == nil {
return nil
}
return state.evidence[start:]
}
func sourceInfosWithVersionEvidence(sources []SourceInfo, evidence []*sourceVersionEvidence, secrets map[string]bool, conservative bool) []SourceInfo {
result := cloneSourceInfos(sources)
if len(evidence) == 0 {