From f7624ae92a292e85bd56d8cfc28109588b79d9f2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 24 Oct 2026 00:05:24 +0000 Subject: [PATCH] Project incoming source destinations through tier operations --- docs/roadmap/remediation.md | 9 +- internal/engine/load_request.go | 16 ++- internal/engine/overlay.go | 21 +++- .../sequence_destination_projection_test.go | 102 ++++++++++++++++++ .../engine/sequence_history_projection.go | 35 +++++- internal/engine/source_version.go | 7 ++ 6 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 internal/engine/sequence_destination_projection_test.go diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 68153c8..9a9ff2e 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -138,7 +138,14 @@ agree with structural assembly before real decoding reconciliation. ## 4. Reproject Nested Source Destinations After Cross-Tier Item Movement — P2 -Status: [ ] Open. +Status: [x] Complete. + +Completion: layered assembly projects incoming source destinations and version +classification evidence with the same item assignments used for values/history. +Earlier contributions retain their historical coordinates, physical identities +and contexts. Main/underlay regressions cover reordered upserts, append, replace, +nested helpers, and source-local version evidence. Verified focused destination, +origin and source-version tests, focused race coverage, and `./scripts/check.sh`. Owners: [`projectAcquiredDestinations`](../../internal/engine/imports.go) and the sequence-operation handoff in [`assembleLayeredInput`](../../internal/engine/load_request.go). diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index f943fec..0429e38 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -93,7 +93,10 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) { var assembly *assemblyAccumulator + mainSourceCount := len(input.sources) + mainEvidence := input.versionState.evidenceSince(0) for index, group := range request.underlays { + evidenceStart := len(input.versionState.evidenceSince(0)) acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy) mergeResourceContexts(&input.contexts, acquired.contexts) if len(diagnostics) != 0 { @@ -105,6 +108,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a lower = assembly.current.collectionPolicies } composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, lower) + sourceStart := len(input.sources) input.sources = append(input.sources, acquired.sources...) if len(diagnostics) != 0 { return input, diagnostics @@ -117,12 +121,14 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a for _, scope := range composition.acquisition { declarations = append(declarations, acquiredSequenceDeclarations(scope)...) } - if diagnostics := assembly.applyOverlayWithCollectionPolicy( + operations, diagnostics := assembly.applyOverlayWithCollectionProjection( composition.root, composition.origins, composition.collectionPolicies, composition.applications, LayerUnderlay, request.sequencePolicy, declarations, - ); len(diagnostics) != 0 { + ) + if len(diagnostics) != 0 { return input, diagnostics } + projectTierSourceDestinations(input.sources[sourceStart:], input.versionState.evidenceSince(evidenceStart), operations) } } } @@ -159,12 +165,14 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a for _, scope := range mainComposition.acquisition { declarations = append(declarations, acquiredSequenceDeclarations(scope)...) } - if diagnostics := assembly.applyOverlayWithCollectionPolicy( + operations, diagnostics := assembly.applyOverlayWithCollectionProjection( mainComposition.root, mainComposition.origins, mainComposition.collectionPolicies, mainComposition.applications, LayerBase, request.sequencePolicy, declarations, - ); len(diagnostics) != 0 { + ) + if len(diagnostics) != 0 { return input, diagnostics } + projectTierSourceDestinations(input.sources[:mainSourceCount], mainEvidence, operations) } } if assembly == nil && activeOverlay(request.overlay) { diff --git a/internal/engine/overlay.go b/internal/engine/overlay.go index d076fb2..c1c1d61 100644 --- a/internal/engine/overlay.go +++ b/internal/engine/overlay.go @@ -65,13 +65,28 @@ func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy( sequencePolicy sequencePolicy, declarations []sequenceDeclarationSpec, ) []Diagnostic { + _, diagnostics := a.applyOverlayWithCollectionProjection(overlay, incomingOrigins, incomingPolicies, applications, layer, sequencePolicy, declarations) + return diagnostics +} + +// applyOverlayWithCollectionProjection also returns the incoming tier's item +// assignments so acquisition evidence can follow the same operation as values. +func (a *assemblyAccumulator) applyOverlayWithCollectionProjection( + overlay *document.Node, + incomingOrigins map[string]Origin, + incomingPolicies collectionPolicyScope, + applications []historyApplication, + layer Layer, + sequencePolicy sequencePolicy, + declarations []sequenceDeclarationSpec, +) ([]sequenceHistoryContribution, []Diagnostic) { if a == nil || a.current == nil || a.current.root == nil || a.current.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping { - return invalidOverlayDocument() + return nil, invalidOverlayDocument() } mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer)) sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, incomingPolicies, layer, sequencePolicy, declarations) if len(diagnostics) != 0 { - return diagnostics + return nil, diagnostics } if len(applications) == 0 { application := historyApplication{layer: layer, value: overlay, sequences: sequences} @@ -83,7 +98,7 @@ func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy( a.current.applications = append(a.current.applications, application) } } - return nil + return sequences, nil } func mergeOverlayMappingsWithCollectionPolicy( diff --git a/internal/engine/sequence_destination_projection_test.go b/internal/engine/sequence_destination_projection_test.go new file mode 100644 index 0000000..7926708 --- /dev/null +++ b/internal/engine/sequence_destination_projection_test.go @@ -0,0 +1,102 @@ +package engine + +import ( + "fmt" + "strings" + "testing" + "testing/fstest" +) + +func TestLayeredSequenceProjectsNestedSourceDestinations(t *testing.T) { + t.Parallel() + type catalog struct { + Items []map[string]any `yaml:"items"` + } + for _, mode := range []string{"upsert", "append", "replace"} { + for _, underlay := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/underlay=%t", mode, underlay), func(t *testing.T) { + files := fstest.MapFS{} + for _, id := range []string{"a", "b", "c"} { + files[id+"/record.yml"] = &fstest.MapFile{Data: []byte("configkit: {imports: [{path: helper.yml}]}\nid: " + id + "\n")} + files[id+"/helper.yml"] = &fstest.MapFile{Data: []byte("value: " + id + "\n")} + } + files["main.yml"] = &fstest.MapFile{Data: []byte("configkit:\n sequences: [{destination: [items], key: id, mode: " + mode + "}]\n imports: [{glob: '*/record.yml', destination: [items], placement: sequence-item}]\n")} + lower := "items: [{id: b}, {id: a}, {id: keep}]\n" + want := map[string]int{"a": 1, "b": 0, "c": 3} + if mode == "append" { + lower = "items: [{id: keep}]\n" + want = map[string]int{"a": 1, "b": 2, "c": 3} + } else if mode == "replace" { + want = map[string]int{"a": 0, "b": 1, "c": 2} + } + incoming := loadInput{source: NewFilesystem("catalog", files).File("main.yml"), imports: true} + request := loadRequest[catalog]{main: incoming, underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte(lower))}}}}} + if underlay { + request.main = loadInput{source: NoSource()} + request.underlays = append(request.underlays, loadGroup{inputs: []loadInput{incoming}}) + } + loaded, err := testGroupedLoad(request) + if err != nil { + t.Fatal(err) + } + helpers := 0 + for _, source := range loaded.Sources() { + if !strings.HasSuffix(source.Path, "/helper.yml") { + continue + } + helpers++ + context, ok := loaded.SourceContext(source.ID) + if !ok { + t.Fatal("missing declaring context") + } + index, ok := want[context.BaseDir] + if !ok || source.Destination.String() != Keys("items").Index(index).String() { + t.Errorf("%s destination = %s, want item %d", context.BaseDir, source.Destination, index) + } + if loaded.Config.Items[index]["value"] != context.BaseDir { + t.Errorf("source destination identifies another record: %#v", loaded.Config.Items[index]) + } + } + if helpers != 3 { + t.Fatalf("retained %d helpers, want 3", helpers) + } + }) + } + } +} + +func TestTierProjectionPreservesSourceLocalVersionEvidence(t *testing.T) { + t.Parallel() + local := Keys("version") + destination := Keys("items").Index(0).Key("details") + lower := &sourceVersionEvidence{sourceID: "lower", path: local, destination: destination.clone()} + higher := &sourceVersionEvidence{ + sourceID: "higher", path: local, resolvedPath: local, + destination: destination.clone(), version: Version{Text: "accepted"}, + projectedPaths: []Path{appendDestination(destination, local), destination.clone()}, + } + sources := []SourceInfo{{ID: "higher", Path: "record/helper.yml", Destination: destination.clone()}} + state := &sourceVersionState{evidence: []*sourceVersionEvidence{lower, higher}} + projectTierSourceDestinations(sources, state.evidenceSince(1), []sequenceHistoryContribution{{ + path: Keys("items"), mode: sequenceHistoryMapped, + items: []sequenceItemProjection{{inputIndex: 0, outputIndex: 1}}, + }}) + want := Keys("items").Index(1).Key("details") + if sources[0].Destination.String() != want.String() || sources[0].Path != "record/helper.yml" || higher.destination.String() != want.String() { + t.Fatalf("projected source/evidence = %#v, %#v", sources, higher) + } + if lower.destination.String() != destination.String() || lower.projectedPaths != nil { + t.Fatal("incoming projection relabeled a lower historical contribution") + } + if higher.path.String() != local.String() || higher.resolvedPath.String() != local.String() { + t.Fatal("projection changed source-local version coordinates") + } + paths := higher.sensitivityPaths() + if len(paths) != 2 || paths[0].String() != want.Key("version").String() || paths[1].String() != want.String() { + t.Fatalf("projected sensitivity paths = %#v", paths) + } + check := sourceVersionCheckForEvidence(higher, newDiffSensitivity(map[string]bool{want.String(): true}, nil), false) + if !check.Redacted || check.Version.Text != "" || check.Path.String() != local.String() { + t.Fatalf("version classification did not use effective coordinates: %#v", check) + } +} diff --git a/internal/engine/sequence_history_projection.go b/internal/engine/sequence_history_projection.go index b3422f9..ac3d362 100644 --- a/internal/engine/sequence_history_projection.go +++ b/internal/engine/sequence_history_projection.go @@ -16,7 +16,7 @@ func projectTierHistoryApplications(applications []historyApplication, operation projected := cloneHistoryApplications(applications) for index, original := range applications { application := &projected[index] - if path, found := projectSequenceHistoryPath(original.path, projections); found { + if path, found := projectSequenceOperationPath(original.path, projections); found { application.path = path } for sequenceIndex, source := range original.sequences { @@ -55,7 +55,7 @@ func projectTierHistoryApplications(applications []historyApplication, operation return projected } -func projectSequenceHistoryPath(path Path, projections map[string]sequenceHistoryContributionIndex) (Path, bool) { +func projectSequenceOperationPath(path Path, projections map[string]sequenceHistoryContributionIndex) (Path, bool) { projected := path.clone() for index, segment := range path.segments { if segment.kind != pathSegmentIndex { @@ -74,3 +74,34 @@ func projectSequenceHistoryPath(path Path, projections map[string]sequenceHistor } return projected, true } + +// Project only incoming sources. Lower/displaced sources retain the coordinates +// of their own historical contribution, rather than becoming incoming survivors. +func projectTierSourceDestinations(sources []SourceInfo, versions []*sourceVersionEvidence, operations []sequenceHistoryContribution) { + if len(operations) == 0 { + return + } + projections := indexSequenceHistoryContributions(operations) + for index := range sources { + if path, found := projectSequenceOperationPath(sources[index].Destination, projections); found { + sources[index].Destination = path + } + } + for _, evidence := range versions { + if evidence == nil { + continue + } + paths := evidence.sensitivityPaths() + projected := make([]Path, len(paths)) + for index, path := range paths { + projected[index] = path.clone() + if destination, found := projectSequenceOperationPath(path, projections); found { + projected[index] = destination + } + } + evidence.projectedPaths = projected + if path, found := projectSequenceOperationPath(evidence.destination, projections); found { + evidence.destination = path + } + } +} diff --git a/internal/engine/source_version.go b/internal/engine/source_version.go index aae1abb..a1c368a 100644 --- a/internal/engine/source_version.go +++ b/internal/engine/source_version.go @@ -53,6 +53,13 @@ type sourceVersionState struct { diagnostics []Diagnostic } +func (state *sourceVersionState) evidenceSince(start int) []*sourceVersionEvidence { + if state == nil { + return nil + } + return state.evidence[start:] +} + func sourceInfosWithVersionEvidence(sources []SourceInfo, evidence []*sourceVersionEvidence, secrets map[string]bool, conservative bool) []SourceInfo { result := cloneSourceInfos(sources) if len(evidence) == 0 {