Preserve inherited source origins and transfer evidence

docs/roadmap/implementation.md

@@ -182,6 +182,15 @@ boundaries. Run the everyday checker.
 
 ## Stage 4 — Preserve derivation, presence, and resource origins
 
+**Completed 2026-09-29.** Private expansion now returns the effective tree,
+per-path origins, shared linked transfer traces, control records, and displaced
+replacement evidence. Contributor records retain original semantic paths and
+source locations through chains, nested destination scopes, composite mappings,
+and sequence descendants; direct child replacements have independent effective
+origins. Clones detach mutable slices and paths, and origin containment checks
+include derivation. Focused source/context, provenance, replacement, and
+detachment tests and `./scripts/check.sh` passed. Public loading remains unchanged.
+
 Implement the selected metadata representation privately in the engine. Carry
 original semantic contributor paths, locations/layers, transfer edges, and control
 evidence through copies and recursive combination. Preserve composite origins and

internal/engine/inheritance.go

@@ -24,9 +24,35 @@ type inheritanceControl struct {
 }
 
 type inheritanceScope struct {
-	root      *document.Node
+	root         *document.Node
-	transfers []inheritanceTransfer
+	origins      map[string]Origin
-	controls  []inheritanceControl
+	transfers    []inheritanceTransfer
+	controls     []inheritanceControl
+	replacements []inheritanceReplacement
+}
+
+// A replacement retains the displaced source/derivation record while the
+// effective origin belongs to the child. Value trees are borrowed immutable
+// inputs or detached templates and are budgeted with retained history later.
+type inheritanceReplacement struct {
+	path      Path
+	before    *document.Node
+	after     *document.Node
+	previous  []displacedDefinition
+	effective Origin
+}
+
+// A link represents one shared relationship edge. Each contributor retains
+// its original semantic path independently of source line/column coordinates.
+type inheritanceTrace struct {
+	previous *inheritanceTrace
+	transfer *inheritanceTransfer
+}
+
+type inheritanceContributor struct {
+	source   Path
+	location Location
+	trace    *inheritanceTrace
 }
 
 type siblingFrame struct {
@@ -44,6 +70,10 @@ func expandSiblingDefinitions(scope *document.Node, scopePath Path) (*inheritanc
 }
 
 func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
+	return expandSiblingDefinitionsWithOrigins(scope, scopePath, limits, nil)
+}
+
+func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin) (*inheritanceScope, []Diagnostic) {
 	if scope == nil || scope.Kind != document.Mapping {
 		return nil, []Diagnostic{{
 			Code: CodeInheritanceInvalid, Severity: SeverityError, Path: scopePath,
@@ -60,6 +90,7 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li
 	slices.Sort(names)
 
 	resolved := make(map[string]*document.Node, len(byName))
+	resolvedOrigins := make(map[string]map[string]Origin, len(byName))
 	state := make(map[string]uint8, len(byName))
 	result := &inheritanceScope{}
 	retained := 1 // scope mapping node
@@ -128,6 +159,7 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li
 			}
 
 			payload := withoutInheritanceDirective(entry.Value)
+			childOrigins := directInheritanceOrigins(payload, path, inputOrigins)
 			if limits.nodes >= 0 {
 				remaining := limits.nodes - retained - 1 // the sibling key
 				within := false
@@ -143,17 +175,37 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li
 			if frame.hasParent {
 				parent := resolved[frame.parent]
 				merged := document.Clone(parent)
-				mergeRecursiveMappings(merged, payload, path, new(mappingIndex), recursiveMergeHooks{})
+				control := inheritanceDirectiveLocation(entry.Value)
+				transfer := inheritanceTransfer{
+					from: scopePath.Key(frame.parent), to: path,
+					reference: path.Key(inheritanceKey), locations: []Location{control},
+				}
+				mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[frame.parent], transfer.from, path, transfer)
+				mergeInheritanceOrigin(mergedOrigins, path, childOrigins[path.String()])
+				mergeRecursiveMappings(merged, payload, path, new(mappingIndex), recursiveMergeHooks{
+					added: func(childPath Path, incoming *document.Node) {
+						copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
+					},
+					combined: func(childPath Path, _ *document.Node) {
+						mergeInheritanceOrigin(mergedOrigins, childPath, childOrigins[childPath.String()])
+					},
+					replaced: func(childPath Path, existing, incoming *document.Node) {
+						previous := displacedOrigins(existing, mergedOrigins, childPath)
+						result.replacements = append(result.replacements, inheritanceReplacement{
+							path: childPath.clone(), before: existing, after: incoming,
+							previous: previous, effective: cloneOrigin(childOrigins[childPath.String()]),
+						})
+						deleteOriginSubtree(mergedOrigins, childPath, existing)
+						copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath)
+					},
+				})
 				merged.Location = entry.Value.Location
 				resolved[frame.name] = merged
-				reference := path.Key(inheritanceKey)
+				resolvedOrigins[frame.name] = mergedOrigins
-				control := inheritanceDirectiveLocation(entry.Value)
+				result.transfers = append(result.transfers, transfer)
-				result.transfers = append(result.transfers, inheritanceTransfer{
-					from: scopePath.Key(frame.parent), to: path,
-					reference: reference, locations: []Location{control},
-				})
 			} else {
 				resolved[frame.name] = document.Clone(payload)
+				resolvedOrigins[frame.name] = childOrigins
 			}
 			count, _ := inheritanceLowerBoundCount(resolved[frame.name], limits.nodes)
 			retained += 1 + count
@@ -164,12 +216,20 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li
 
 	root := *scope
 	root.Mapping = make([]document.Entry, len(scope.Mapping))
+	rootOrigin := directInheritanceOrigins(&document.Node{Kind: document.Mapping, Location: scope.Location}, scopePath, inputOrigins)
+	result.origins = rootOrigin
 	for index, entry := range scope.Mapping {
 		root.Mapping[index] = entry
 		if resolved[entry.Key] != nil {
 			root.Mapping[index].Value = resolved[entry.Key]
+			for pointer, origin := range resolvedOrigins[entry.Key] {
+				result.origins[pointer] = origin
+			}
 		} else {
 			root.Mapping[index].Value = document.Clone(entry.Value)
+			for pointer, origin := range directInheritanceOrigins(entry.Value, scopePath.Key(entry.Key), inputOrigins) {
+				result.origins[pointer] = origin
+			}
 		}
 	}
 	result.root = &root

internal/engine/inheritance_origin.go

@@ -0,0 +1,127 @@
+package engine
+
+import (
+	"slices"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func directInheritanceOrigins(node *document.Node, root Path, input map[string]Origin) map[string]Origin {
+	result := make(map[string]Origin)
+	walkDocument(node, root, func(path Path, value *document.Node) {
+		origin, found := input[path.String()]
+		if !found {
+			origin = Origin{Layer: LayerBase, Locations: []Location{publicLocation(value.Location)}}
+		}
+		origin = cloneOrigin(origin)
+		if len(origin.contributors) == 0 {
+			origin.contributors = make([]inheritanceContributor, len(origin.Locations))
+			for index, location := range origin.Locations {
+				origin.contributors[index] = inheritanceContributor{
+					source: path.clone(), location: location,
+				}
+			}
+		}
+		result[path.String()] = origin
+	})
+	return result
+}
+
+func transferredInheritanceOrigins(parent map[string]Origin, from, to Path, control inheritanceTransfer) map[string]Origin {
+	result := make(map[string]Origin, len(parent))
+	transfer := control // one immutable edge shared by every matching contributor
+	links := make(map[*inheritanceTrace]*inheritanceTrace)
+	for pointer, origin := range parent {
+		path, err := ParsePath(pointer)
+		if err != nil {
+			continue
+		}
+		relative, ok := relativeHistoryPath(from, path)
+		if !ok {
+			continue
+		}
+		destination := appendInheritancePath(to, relative)
+		origin = cloneOrigin(origin)
+		for index, contributor := range origin.contributors {
+			link, found := links[contributor.trace]
+			if !found {
+				link = &inheritanceTrace{previous: contributor.trace, transfer: &transfer}
+				links[contributor.trace] = link
+			}
+			origin.contributors[index].trace = link
+		}
+		result[destination.String()] = origin
+	}
+	return result
+}
+
+func appendInheritancePath(root, relative Path) Path {
+	return Path{segments: append(clonePathSegments(root.segments, len(relative.segments)), relative.segments...)}
+}
+
+func mergeInheritanceOrigin(origins map[string]Origin, path Path, incoming Origin) {
+	pointer := path.String()
+	current := origins[pointer]
+	current.Layer = incoming.Layer
+	current.Locations = append(current.Locations, incoming.Locations...)
+	current.contributors = append(current.contributors, incoming.contributors...)
+	origins[pointer] = current
+}
+
+func copyInheritanceSubtree(target, incoming map[string]Origin, node *document.Node, root Path) {
+	walkDocument(node, root, func(path Path, _ *document.Node) {
+		target[path.String()] = cloneOrigin(incoming[path.String()])
+	})
+}
+
+// Materialize concrete steps only for an inspection record. The linked private
+// traces remain compact when one parent supplies many descendant values.
+func projectInheritanceSteps(origin Origin) []InheritanceStep {
+	steps := cloneInheritanceSteps(origin.Inheritance)
+	for _, contributor := range origin.contributors {
+		var reverse []*inheritanceTrace
+		for link := contributor.trace; link != nil; link = link.previous {
+			reverse = append(reverse, link)
+		}
+		current := contributor.source
+		for index := len(reverse) - 1; index >= 0; index-- {
+			transfer := reverse[index].transfer
+			relative, ok := relativeHistoryPath(transfer.from, current)
+			if !ok {
+				continue
+			}
+			next := appendInheritancePath(transfer.to, relative)
+			step := InheritanceStep{
+				From: current.clone(), To: next.clone(), Reference: transfer.reference.clone(),
+				Locations: cloneLocations(transfer.locations),
+			}
+			if !slices.ContainsFunc(steps, func(existing InheritanceStep) bool {
+				return existing.From.String() == step.From.String() &&
+					existing.To.String() == step.To.String() &&
+					existing.Reference.String() == step.Reference.String() &&
+					slices.Equal(existing.Locations, step.Locations)
+			}) {
+				steps = append(steps, step)
+			}
+			current = next
+		}
+	}
+	return steps
+}
+
+func sameInheritanceTrace(left, right *inheritanceTrace) bool {
+	for left != nil && right != nil {
+		if left == right {
+			return true
+		}
+		l, r := left.transfer, right.transfer
+		if !slices.Equal(l.from.segments, r.from.segments) ||
+			!slices.Equal(l.to.segments, r.to.segments) ||
+			!slices.Equal(l.reference.segments, r.reference.segments) ||
+			!slices.Equal(l.locations, r.locations) {
+			return false
+		}
+		left, right = left.previous, right.previous
+	}
+	return left == nil && right == nil
+}

internal/engine/inheritance_origin_test.go

@@ -0,0 +1,141 @@
+package engine
+
+import (
+	"io/fs"
+	"testing"
+	"testing/fstest"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func TestInheritedOriginsKeepSourcePathsPresenceAndIndependentReplacement(t *testing.T) {
+	parent := parseCompositionDocument(t, "source-1", `parent:
+  enabled: false
+  count: 0
+  optional: null
+  items: [{file: message.txt}]
+  settings: {shared: parent}
+`)
+	child := parseCompositionDocument(t, "source-2", `child:
+  extends: parent
+  count: 3
+  settings: {own: child}
+`)
+	composed, diagnostics := composeDocuments(parent, child)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	result, diagnostics := expandInheritanceWithOrigins(composed.root, composed.origins, defaultProjectionLimits())
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	for _, key := range []string{"enabled", "optional"} {
+		path := Keys("child", key)
+		value, found := documentAtPath(result.root, path)
+		origin, hasOrigin := result.origins[path.String()]
+		if !found || !hasOrigin || value == nil || len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-1" ||
+			len(projectInheritanceSteps(origin)) != 1 {
+			t.Fatalf("inherited %s = %#v, %#v", key, value, origin)
+		}
+	}
+	count := result.origins["/child/count"]
+	if len(count.Locations) != 1 || count.Locations[0].SourceID != "source-2" || len(projectInheritanceSteps(count)) != 0 {
+		t.Fatalf("direct replacement origin = %#v", count)
+	}
+	if len(result.replacements) != 1 || result.replacements[0].path.String() != "/child/count" ||
+		result.replacements[0].before.Text != "0" || result.replacements[0].after.Text != "3" ||
+		len(result.replacements[0].previous) != 1 ||
+		len(projectInheritanceSteps(result.replacements[0].previous[0].origin)) != 1 {
+		t.Fatalf("displaced inheritance = %#v", result.replacements)
+	}
+	settings := result.origins["/child/settings"]
+	if len(settings.Locations) != 2 || settings.Locations[0].SourceID != "source-1" ||
+		settings.Locations[1].SourceID != "source-2" || len(projectInheritanceSteps(settings)) != 1 {
+		t.Fatalf("composite mapping origin = %#v", settings)
+	}
+	if originContainedBy(result.origins["/child/enabled"], result.origins["/parent/enabled"]) ||
+		originContainedBy(result.origins["/parent/enabled"], result.origins["/child/enabled"]) {
+		t.Fatal("direct and inherited evidence collapsed despite matching source coordinates")
+	}
+	original := result.origins["/child/enabled"]
+	detached := cloneOrigin(original)
+	detached.Locations[0].SourceID = "changed"
+	detached.contributors[0].source.segments[0].key = "changed"
+	stepsCopy := projectInheritanceSteps(original)
+	stepsCopy[0].Locations[0].SourceID = "changed"
+	if original.Locations[0].SourceID != "source-1" ||
+		original.contributors[0].source.String() != "/parent/enabled" ||
+		projectInheritanceSteps(original)[0].Locations[0].SourceID != "source-2" {
+		t.Fatal("cloned origin or projected steps changed retained evidence")
+	}
+	resource := result.origins["/child/items/0/file"]
+	steps := projectInheritanceSteps(resource)
+	if len(steps) != 1 || steps[0].From.String() != "/parent/items/0/file" ||
+		steps[0].To.String() != "/child/items/0/file" || steps[0].Reference.String() != "/child/extends" ||
+		len(resource.Locations) != 1 || resource.Locations[0].SourceID != "source-1" {
+		t.Fatalf("sequence resource origin = %#v, steps = %#v", resource, steps)
+	}
+	contexts := map[string]fs.FS{
+		"source-1": fstest.MapFS{"message.txt": &fstest.MapFile{Data: []byte("from parent")}},
+		"source-2": fstest.MapFS{"message.txt": &fstest.MapFile{Data: []byte("from child")}},
+	}
+	data, err := fs.ReadFile(contexts[resource.Locations[0].SourceID], "message.txt")
+	if err != nil || string(data) != "from parent" {
+		t.Fatalf("inherited resource = %q, %v", data, err)
+	}
+}
+
+func TestInheritedOriginsRetainChainsAndNestedDestinations(t *testing.T) {
+	first := parseCompositionDocument(t, "source-1", `first:
+  enabled: false
+  settings:
+    primary: {value: original}
+    secondary: {extends: primary}
+`)
+	second := parseCompositionDocument(t, "source-2", `first: {other: present}
+middle: {extends: first}
+last:
+  extends: middle
+  settings:
+    primary: {value: replaced}
+`)
+	composed, diagnostics := composeDocuments(first, second)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	result, diagnostics := expandInheritanceWithOrigins(composed.root, composed.origins, defaultProjectionLimits())
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	chain := projectInheritanceSteps(result.origins["/last/enabled"])
+	if len(chain) != 2 || chain[0].From.String() != "/first/enabled" ||
+		chain[0].To.String() != "/middle/enabled" || chain[1].From.String() != "/middle/enabled" ||
+		chain[1].To.String() != "/last/enabled" {
+		t.Fatalf("chain = %#v", chain)
+	}
+	composite := result.origins["/last"]
+	if len(composite.Locations) < 2 || len(composite.contributors) != len(composite.Locations) {
+		t.Fatalf("composite contributors = %#v", composite)
+	}
+	seenSources := make(map[string]bool)
+	for _, contributor := range composite.contributors {
+		seenSources[contributor.location.SourceID] = true
+	}
+	if !seenSources["source-1"] || !seenSources["source-2"] {
+		t.Fatalf("same-layer source contributors = %#v", composite.contributors)
+	}
+	nested := projectInheritanceSteps(result.origins["/last/settings/secondary/value"])
+	if len(nested) != 1 || nested[0].From.String() != "/last/settings/primary/value" ||
+		nested[0].To.String() != "/last/settings/secondary/value" {
+		t.Fatalf("destination-specific nested transfer = %#v", nested)
+	}
+	if got := result.origins["/last/settings/secondary/value"].Locations[0].SourceID; got != "source-2" {
+		t.Fatalf("nested replacement source = %q", got)
+	}
+	if original, found := documentAtPath(first, Keys("first", "settings", "primary", "value")); !found || original.Text != "original" {
+		t.Fatal("expansion changed its input")
+	}
+	if inherited, found := documentAtPath(result.root, Keys("last", "settings", "secondary", "value")); !found || inherited.Kind != document.String || inherited.Text != "replaced" {
+		t.Fatalf("nested inherited value = %#v, %v", inherited, found)
+	}
+}

internal/engine/inheritance_walk.go

@@ -8,34 +8,44 @@ import (
 )
 
 type inheritanceWalker struct {
-	root      **document.Node
+	root         **document.Node
-	limits    projectionLimits
+	origins      map[string]Origin
-	transfers []inheritanceTransfer
+	limits       projectionLimits
-	controls  []inheritanceControl
+	transfers    []inheritanceTransfer
+	controls     []inheritanceControl
+	replacements []inheritanceReplacement
 }
 
 // expandInheritanceTree resolves each destination scope after its enclosing
 // relationships. It remains a private, test-only operation until the loading
 // lifecycle also carries its complete provenance and inspection evidence.
 func expandInheritanceTree(input *document.Node, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
+	return expandInheritanceWithOrigins(input, nil, limits)
+}
+
+func expandInheritanceWithOrigins(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
 	if input == nil || input.Kind != document.Mapping {
 		return nil, []Diagnostic{{Code: CodeInheritanceInvalid, Severity: SeverityError,
 			Message: "inheritance requires a mapping document root"}}
 	}
 	root := input
-	walker := inheritanceWalker{root: &root, limits: limits}
+	walker := inheritanceWalker{root: &root, limits: limits, origins: cloneOriginIndex(inputOrigins)}
 	if problem := walker.visit(&root, Path{}, true); problem != nil {
 		return nil, []Diagnostic{*problem}
 	}
 	if problem := inheritanceFinalBudget(root, limits); problem != nil {
 		return nil, []Diagnostic{*problem}
 	}
-	return &inheritanceScope{root: root, transfers: walker.transfers, controls: walker.controls}, nil
+	return &inheritanceScope{
+		root: root, origins: walker.origins, transfers: walker.transfers,
+		controls: walker.controls, replacements: walker.replacements,
+	}, nil
 }
 
 func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool) *Diagnostic {
 	switch (*node).Kind {
 	case document.Mapping:
+		original := *node
 		if unscoped {
 			reference, found := inheritanceDirective(*node)
 			if found {
@@ -49,16 +59,22 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool
 				w.controls = append(w.controls, inheritanceControl{
 					path: referencePath, locations: []Location{location}, cancelled: true,
 				})
+				deleteOriginSubtree(w.origins, referencePath, reference.Value)
 				*node = withoutInheritanceDirective(*node)
 			}
 		}
-		resolved, diagnostics := expandSiblingDefinitionsWithLimits(*node, path, w.limits)
+		resolved, diagnostics := expandSiblingDefinitionsWithOrigins(*node, path, w.limits, w.origins)
 		if len(diagnostics) != 0 {
 			return &diagnostics[0]
 		}
 		*node = resolved.root
+		deleteOriginSubtree(w.origins, path, original)
+		for pointer, origin := range resolved.origins {
+			w.origins[pointer] = origin
+		}
 		w.transfers = append(w.transfers, resolved.transfers...)
 		w.controls = append(w.controls, resolved.controls...)
+		w.replacements = append(w.replacements, resolved.replacements...)
 		if !inheritanceLowerBoundWithin(*w.root, w.limits.nodes) {
 			problem := inheritanceBudgetDiagnostic(path, (*node).Location)
 			return &problem

internal/engine/load_metadata.go

@@ -201,6 +201,23 @@ func originContainedBy(candidate, container Origin) bool {
 			return false
 		}
 	}
+	if (len(candidate.contributors) == 0) != (len(container.contributors) == 0) {
+		return false
+	}
+	for _, contributor := range candidate.contributors {
+		found := false
+		for _, available := range container.contributors {
+			if contributor.location == available.location &&
+				slices.Equal(contributor.source.segments, available.source.segments) &&
+				sameInheritanceTrace(contributor.trace, available.trace) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
 	return true
 }
 

internal/engine/records.go

@@ -201,13 +201,25 @@ type Diagnostic struct {
 	Related  []Location
 }
 
+// InheritanceStep identifies a concrete payload transfer controlled by a
+// relationship directive. Locations belong to that directive, not its value.
+type InheritanceStep struct {
+	From      Path
+	To        Path
+	Reference Path
+	Locations []Location
+}
+
 // Origin describes where an effective definition came from. Multiple locations
 // represent a composite origin; resolution origins are synthetic and identify their
-// operation instead of a file location.
+// operation instead of a file location. Inheritance is projected from private
+// contributor traces at inspection boundaries.
 type Origin struct {
-	Layer     Layer
+	Layer        Layer
-	Locations []Location
+	Locations    []Location
-	Operation ChangeKind
+	Operation    ChangeKind
+	Inheritance  []InheritanceStep
+	contributors []inheritanceContributor
 }
 
 // FieldInfo describes an effective configuration path.
@@ -261,9 +273,32 @@ func cloneSourceInfos(sources []SourceInfo) []SourceInfo {
 
 func cloneOrigin(origin Origin) Origin {
 	origin.Locations = cloneLocations(origin.Locations)
+	origin.Inheritance = cloneInheritanceSteps(origin.Inheritance)
+	if origin.contributors != nil {
+		contributors := make([]inheritanceContributor, len(origin.contributors))
+		for index, contributor := range origin.contributors {
+			contributors[index] = contributor
+			contributors[index].source = contributor.source.clone()
+		}
+		origin.contributors = contributors
+	}
 	return origin
 }
 
+func cloneInheritanceSteps(steps []InheritanceStep) []InheritanceStep {
+	if steps == nil {
+		return nil
+	}
+	cloned := make([]InheritanceStep, len(steps))
+	for index, step := range steps {
+		cloned[index] = InheritanceStep{
+			From: step.From.clone(), To: step.To.clone(), Reference: step.Reference.clone(),
+			Locations: cloneLocations(step.Locations),
+		}
+	}
+	return cloned
+}
+
 func cloneFieldInfo(info FieldInfo) FieldInfo {
 	info.Origin = cloneOrigin(info.Origin)
 	return info