diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a55b34a..e3d1da9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -182,6 +182,15 @@ boundaries. Run the everyday checker. ## Stage 4 — Preserve derivation, presence, and resource origins +**Completed 2026-09-29.** Private expansion now returns the effective tree, +per-path origins, shared linked transfer traces, control records, and displaced +replacement evidence. Contributor records retain original semantic paths and +source locations through chains, nested destination scopes, composite mappings, +and sequence descendants; direct child replacements have independent effective +origins. Clones detach mutable slices and paths, and origin containment checks +include derivation. Focused source/context, provenance, replacement, and +detachment tests and `./scripts/check.sh` passed. Public loading remains unchanged. + Implement the selected metadata representation privately in the engine. Carry original semantic contributor paths, locations/layers, transfer edges, and control evidence through copies and recursive combination. Preserve composite origins and diff --git a/internal/engine/inheritance.go b/internal/engine/inheritance.go index cdafdb0..7e554b3 100644 --- a/internal/engine/inheritance.go +++ b/internal/engine/inheritance.go @@ -24,9 +24,35 @@ type inheritanceControl struct { } type inheritanceScope struct { - root *document.Node - transfers []inheritanceTransfer - controls []inheritanceControl + root *document.Node + origins map[string]Origin + transfers []inheritanceTransfer + controls []inheritanceControl + replacements []inheritanceReplacement +} + +// A replacement retains the displaced source/derivation record while the +// effective origin belongs to the child. Value trees are borrowed immutable +// inputs or detached templates and are budgeted with retained history later. +type inheritanceReplacement struct { + path Path + before *document.Node + after *document.Node + previous []displacedDefinition + effective Origin +} + +// A link represents one shared relationship edge. Each contributor retains +// its original semantic path independently of source line/column coordinates. +type inheritanceTrace struct { + previous *inheritanceTrace + transfer *inheritanceTransfer +} + +type inheritanceContributor struct { + source Path + location Location + trace *inheritanceTrace } type siblingFrame struct { @@ -44,6 +70,10 @@ func expandSiblingDefinitions(scope *document.Node, scopePath Path) (*inheritanc } func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, limits projectionLimits) (*inheritanceScope, []Diagnostic) { + return expandSiblingDefinitionsWithOrigins(scope, scopePath, limits, nil) +} + +func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin) (*inheritanceScope, []Diagnostic) { if scope == nil || scope.Kind != document.Mapping { return nil, []Diagnostic{{ Code: CodeInheritanceInvalid, Severity: SeverityError, Path: scopePath, @@ -60,6 +90,7 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li slices.Sort(names) resolved := make(map[string]*document.Node, len(byName)) + resolvedOrigins := make(map[string]map[string]Origin, len(byName)) state := make(map[string]uint8, len(byName)) result := &inheritanceScope{} retained := 1 // scope mapping node @@ -128,6 +159,7 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li } payload := withoutInheritanceDirective(entry.Value) + childOrigins := directInheritanceOrigins(payload, path, inputOrigins) if limits.nodes >= 0 { remaining := limits.nodes - retained - 1 // the sibling key within := false @@ -143,17 +175,37 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li if frame.hasParent { parent := resolved[frame.parent] merged := document.Clone(parent) - mergeRecursiveMappings(merged, payload, path, new(mappingIndex), recursiveMergeHooks{}) + control := inheritanceDirectiveLocation(entry.Value) + transfer := inheritanceTransfer{ + from: scopePath.Key(frame.parent), to: path, + reference: path.Key(inheritanceKey), locations: []Location{control}, + } + mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[frame.parent], transfer.from, path, transfer) + mergeInheritanceOrigin(mergedOrigins, path, childOrigins[path.String()]) + mergeRecursiveMappings(merged, payload, path, new(mappingIndex), recursiveMergeHooks{ + added: func(childPath Path, incoming *document.Node) { + copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath) + }, + combined: func(childPath Path, _ *document.Node) { + mergeInheritanceOrigin(mergedOrigins, childPath, childOrigins[childPath.String()]) + }, + replaced: func(childPath Path, existing, incoming *document.Node) { + previous := displacedOrigins(existing, mergedOrigins, childPath) + result.replacements = append(result.replacements, inheritanceReplacement{ + path: childPath.clone(), before: existing, after: incoming, + previous: previous, effective: cloneOrigin(childOrigins[childPath.String()]), + }) + deleteOriginSubtree(mergedOrigins, childPath, existing) + copyInheritanceSubtree(mergedOrigins, childOrigins, incoming, childPath) + }, + }) merged.Location = entry.Value.Location resolved[frame.name] = merged - reference := path.Key(inheritanceKey) - control := inheritanceDirectiveLocation(entry.Value) - result.transfers = append(result.transfers, inheritanceTransfer{ - from: scopePath.Key(frame.parent), to: path, - reference: reference, locations: []Location{control}, - }) + resolvedOrigins[frame.name] = mergedOrigins + result.transfers = append(result.transfers, transfer) } else { resolved[frame.name] = document.Clone(payload) + resolvedOrigins[frame.name] = childOrigins } count, _ := inheritanceLowerBoundCount(resolved[frame.name], limits.nodes) retained += 1 + count @@ -164,12 +216,20 @@ func expandSiblingDefinitionsWithLimits(scope *document.Node, scopePath Path, li root := *scope root.Mapping = make([]document.Entry, len(scope.Mapping)) + rootOrigin := directInheritanceOrigins(&document.Node{Kind: document.Mapping, Location: scope.Location}, scopePath, inputOrigins) + result.origins = rootOrigin for index, entry := range scope.Mapping { root.Mapping[index] = entry if resolved[entry.Key] != nil { root.Mapping[index].Value = resolved[entry.Key] + for pointer, origin := range resolvedOrigins[entry.Key] { + result.origins[pointer] = origin + } } else { root.Mapping[index].Value = document.Clone(entry.Value) + for pointer, origin := range directInheritanceOrigins(entry.Value, scopePath.Key(entry.Key), inputOrigins) { + result.origins[pointer] = origin + } } } result.root = &root diff --git a/internal/engine/inheritance_origin.go b/internal/engine/inheritance_origin.go new file mode 100644 index 0000000..e4e2ba9 --- /dev/null +++ b/internal/engine/inheritance_origin.go @@ -0,0 +1,127 @@ +package engine + +import ( + "slices" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func directInheritanceOrigins(node *document.Node, root Path, input map[string]Origin) map[string]Origin { + result := make(map[string]Origin) + walkDocument(node, root, func(path Path, value *document.Node) { + origin, found := input[path.String()] + if !found { + origin = Origin{Layer: LayerBase, Locations: []Location{publicLocation(value.Location)}} + } + origin = cloneOrigin(origin) + if len(origin.contributors) == 0 { + origin.contributors = make([]inheritanceContributor, len(origin.Locations)) + for index, location := range origin.Locations { + origin.contributors[index] = inheritanceContributor{ + source: path.clone(), location: location, + } + } + } + result[path.String()] = origin + }) + return result +} + +func transferredInheritanceOrigins(parent map[string]Origin, from, to Path, control inheritanceTransfer) map[string]Origin { + result := make(map[string]Origin, len(parent)) + transfer := control // one immutable edge shared by every matching contributor + links := make(map[*inheritanceTrace]*inheritanceTrace) + for pointer, origin := range parent { + path, err := ParsePath(pointer) + if err != nil { + continue + } + relative, ok := relativeHistoryPath(from, path) + if !ok { + continue + } + destination := appendInheritancePath(to, relative) + origin = cloneOrigin(origin) + for index, contributor := range origin.contributors { + link, found := links[contributor.trace] + if !found { + link = &inheritanceTrace{previous: contributor.trace, transfer: &transfer} + links[contributor.trace] = link + } + origin.contributors[index].trace = link + } + result[destination.String()] = origin + } + return result +} + +func appendInheritancePath(root, relative Path) Path { + return Path{segments: append(clonePathSegments(root.segments, len(relative.segments)), relative.segments...)} +} + +func mergeInheritanceOrigin(origins map[string]Origin, path Path, incoming Origin) { + pointer := path.String() + current := origins[pointer] + current.Layer = incoming.Layer + current.Locations = append(current.Locations, incoming.Locations...) + current.contributors = append(current.contributors, incoming.contributors...) + origins[pointer] = current +} + +func copyInheritanceSubtree(target, incoming map[string]Origin, node *document.Node, root Path) { + walkDocument(node, root, func(path Path, _ *document.Node) { + target[path.String()] = cloneOrigin(incoming[path.String()]) + }) +} + +// Materialize concrete steps only for an inspection record. The linked private +// traces remain compact when one parent supplies many descendant values. +func projectInheritanceSteps(origin Origin) []InheritanceStep { + steps := cloneInheritanceSteps(origin.Inheritance) + for _, contributor := range origin.contributors { + var reverse []*inheritanceTrace + for link := contributor.trace; link != nil; link = link.previous { + reverse = append(reverse, link) + } + current := contributor.source + for index := len(reverse) - 1; index >= 0; index-- { + transfer := reverse[index].transfer + relative, ok := relativeHistoryPath(transfer.from, current) + if !ok { + continue + } + next := appendInheritancePath(transfer.to, relative) + step := InheritanceStep{ + From: current.clone(), To: next.clone(), Reference: transfer.reference.clone(), + Locations: cloneLocations(transfer.locations), + } + if !slices.ContainsFunc(steps, func(existing InheritanceStep) bool { + return existing.From.String() == step.From.String() && + existing.To.String() == step.To.String() && + existing.Reference.String() == step.Reference.String() && + slices.Equal(existing.Locations, step.Locations) + }) { + steps = append(steps, step) + } + current = next + } + } + return steps +} + +func sameInheritanceTrace(left, right *inheritanceTrace) bool { + for left != nil && right != nil { + if left == right { + return true + } + l, r := left.transfer, right.transfer + if !slices.Equal(l.from.segments, r.from.segments) || + !slices.Equal(l.to.segments, r.to.segments) || + !slices.Equal(l.reference.segments, r.reference.segments) || + !slices.Equal(l.locations, r.locations) { + return false + } + left, right = left.previous, right.previous + } + return left == nil && right == nil +} diff --git a/internal/engine/inheritance_origin_test.go b/internal/engine/inheritance_origin_test.go new file mode 100644 index 0000000..f2067d5 --- /dev/null +++ b/internal/engine/inheritance_origin_test.go @@ -0,0 +1,141 @@ +package engine + +import ( + "io/fs" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +func TestInheritedOriginsKeepSourcePathsPresenceAndIndependentReplacement(t *testing.T) { + parent := parseCompositionDocument(t, "source-1", `parent: + enabled: false + count: 0 + optional: null + items: [{file: message.txt}] + settings: {shared: parent} +`) + child := parseCompositionDocument(t, "source-2", `child: + extends: parent + count: 3 + settings: {own: child} +`) + composed, diagnostics := composeDocuments(parent, child) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + result, diagnostics := expandInheritanceWithOrigins(composed.root, composed.origins, defaultProjectionLimits()) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + for _, key := range []string{"enabled", "optional"} { + path := Keys("child", key) + value, found := documentAtPath(result.root, path) + origin, hasOrigin := result.origins[path.String()] + if !found || !hasOrigin || value == nil || len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-1" || + len(projectInheritanceSteps(origin)) != 1 { + t.Fatalf("inherited %s = %#v, %#v", key, value, origin) + } + } + count := result.origins["/child/count"] + if len(count.Locations) != 1 || count.Locations[0].SourceID != "source-2" || len(projectInheritanceSteps(count)) != 0 { + t.Fatalf("direct replacement origin = %#v", count) + } + if len(result.replacements) != 1 || result.replacements[0].path.String() != "/child/count" || + result.replacements[0].before.Text != "0" || result.replacements[0].after.Text != "3" || + len(result.replacements[0].previous) != 1 || + len(projectInheritanceSteps(result.replacements[0].previous[0].origin)) != 1 { + t.Fatalf("displaced inheritance = %#v", result.replacements) + } + settings := result.origins["/child/settings"] + if len(settings.Locations) != 2 || settings.Locations[0].SourceID != "source-1" || + settings.Locations[1].SourceID != "source-2" || len(projectInheritanceSteps(settings)) != 1 { + t.Fatalf("composite mapping origin = %#v", settings) + } + if originContainedBy(result.origins["/child/enabled"], result.origins["/parent/enabled"]) || + originContainedBy(result.origins["/parent/enabled"], result.origins["/child/enabled"]) { + t.Fatal("direct and inherited evidence collapsed despite matching source coordinates") + } + original := result.origins["/child/enabled"] + detached := cloneOrigin(original) + detached.Locations[0].SourceID = "changed" + detached.contributors[0].source.segments[0].key = "changed" + stepsCopy := projectInheritanceSteps(original) + stepsCopy[0].Locations[0].SourceID = "changed" + if original.Locations[0].SourceID != "source-1" || + original.contributors[0].source.String() != "/parent/enabled" || + projectInheritanceSteps(original)[0].Locations[0].SourceID != "source-2" { + t.Fatal("cloned origin or projected steps changed retained evidence") + } + resource := result.origins["/child/items/0/file"] + steps := projectInheritanceSteps(resource) + if len(steps) != 1 || steps[0].From.String() != "/parent/items/0/file" || + steps[0].To.String() != "/child/items/0/file" || steps[0].Reference.String() != "/child/extends" || + len(resource.Locations) != 1 || resource.Locations[0].SourceID != "source-1" { + t.Fatalf("sequence resource origin = %#v, steps = %#v", resource, steps) + } + contexts := map[string]fs.FS{ + "source-1": fstest.MapFS{"message.txt": &fstest.MapFile{Data: []byte("from parent")}}, + "source-2": fstest.MapFS{"message.txt": &fstest.MapFile{Data: []byte("from child")}}, + } + data, err := fs.ReadFile(contexts[resource.Locations[0].SourceID], "message.txt") + if err != nil || string(data) != "from parent" { + t.Fatalf("inherited resource = %q, %v", data, err) + } +} + +func TestInheritedOriginsRetainChainsAndNestedDestinations(t *testing.T) { + first := parseCompositionDocument(t, "source-1", `first: + enabled: false + settings: + primary: {value: original} + secondary: {extends: primary} +`) + second := parseCompositionDocument(t, "source-2", `first: {other: present} +middle: {extends: first} +last: + extends: middle + settings: + primary: {value: replaced} +`) + composed, diagnostics := composeDocuments(first, second) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + result, diagnostics := expandInheritanceWithOrigins(composed.root, composed.origins, defaultProjectionLimits()) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + chain := projectInheritanceSteps(result.origins["/last/enabled"]) + if len(chain) != 2 || chain[0].From.String() != "/first/enabled" || + chain[0].To.String() != "/middle/enabled" || chain[1].From.String() != "/middle/enabled" || + chain[1].To.String() != "/last/enabled" { + t.Fatalf("chain = %#v", chain) + } + composite := result.origins["/last"] + if len(composite.Locations) < 2 || len(composite.contributors) != len(composite.Locations) { + t.Fatalf("composite contributors = %#v", composite) + } + seenSources := make(map[string]bool) + for _, contributor := range composite.contributors { + seenSources[contributor.location.SourceID] = true + } + if !seenSources["source-1"] || !seenSources["source-2"] { + t.Fatalf("same-layer source contributors = %#v", composite.contributors) + } + nested := projectInheritanceSteps(result.origins["/last/settings/secondary/value"]) + if len(nested) != 1 || nested[0].From.String() != "/last/settings/primary/value" || + nested[0].To.String() != "/last/settings/secondary/value" { + t.Fatalf("destination-specific nested transfer = %#v", nested) + } + if got := result.origins["/last/settings/secondary/value"].Locations[0].SourceID; got != "source-2" { + t.Fatalf("nested replacement source = %q", got) + } + if original, found := documentAtPath(first, Keys("first", "settings", "primary", "value")); !found || original.Text != "original" { + t.Fatal("expansion changed its input") + } + if inherited, found := documentAtPath(result.root, Keys("last", "settings", "secondary", "value")); !found || inherited.Kind != document.String || inherited.Text != "replaced" { + t.Fatalf("nested inherited value = %#v, %v", inherited, found) + } +} diff --git a/internal/engine/inheritance_walk.go b/internal/engine/inheritance_walk.go index 0469782..5ed66a8 100644 --- a/internal/engine/inheritance_walk.go +++ b/internal/engine/inheritance_walk.go @@ -8,34 +8,44 @@ import ( ) type inheritanceWalker struct { - root **document.Node - limits projectionLimits - transfers []inheritanceTransfer - controls []inheritanceControl + root **document.Node + origins map[string]Origin + limits projectionLimits + transfers []inheritanceTransfer + controls []inheritanceControl + replacements []inheritanceReplacement } // expandInheritanceTree resolves each destination scope after its enclosing // relationships. It remains a private, test-only operation until the loading // lifecycle also carries its complete provenance and inspection evidence. func expandInheritanceTree(input *document.Node, limits projectionLimits) (*inheritanceScope, []Diagnostic) { + return expandInheritanceWithOrigins(input, nil, limits) +} + +func expandInheritanceWithOrigins(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits) (*inheritanceScope, []Diagnostic) { if input == nil || input.Kind != document.Mapping { return nil, []Diagnostic{{Code: CodeInheritanceInvalid, Severity: SeverityError, Message: "inheritance requires a mapping document root"}} } root := input - walker := inheritanceWalker{root: &root, limits: limits} + walker := inheritanceWalker{root: &root, limits: limits, origins: cloneOriginIndex(inputOrigins)} if problem := walker.visit(&root, Path{}, true); problem != nil { return nil, []Diagnostic{*problem} } if problem := inheritanceFinalBudget(root, limits); problem != nil { return nil, []Diagnostic{*problem} } - return &inheritanceScope{root: root, transfers: walker.transfers, controls: walker.controls}, nil + return &inheritanceScope{ + root: root, origins: walker.origins, transfers: walker.transfers, + controls: walker.controls, replacements: walker.replacements, + }, nil } func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool) *Diagnostic { switch (*node).Kind { case document.Mapping: + original := *node if unscoped { reference, found := inheritanceDirective(*node) if found { @@ -49,16 +59,22 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool w.controls = append(w.controls, inheritanceControl{ path: referencePath, locations: []Location{location}, cancelled: true, }) + deleteOriginSubtree(w.origins, referencePath, reference.Value) *node = withoutInheritanceDirective(*node) } } - resolved, diagnostics := expandSiblingDefinitionsWithLimits(*node, path, w.limits) + resolved, diagnostics := expandSiblingDefinitionsWithOrigins(*node, path, w.limits, w.origins) if len(diagnostics) != 0 { return &diagnostics[0] } *node = resolved.root + deleteOriginSubtree(w.origins, path, original) + for pointer, origin := range resolved.origins { + w.origins[pointer] = origin + } w.transfers = append(w.transfers, resolved.transfers...) w.controls = append(w.controls, resolved.controls...) + w.replacements = append(w.replacements, resolved.replacements...) if !inheritanceLowerBoundWithin(*w.root, w.limits.nodes) { problem := inheritanceBudgetDiagnostic(path, (*node).Location) return &problem diff --git a/internal/engine/load_metadata.go b/internal/engine/load_metadata.go index 8219430..93227ed 100644 --- a/internal/engine/load_metadata.go +++ b/internal/engine/load_metadata.go @@ -201,6 +201,23 @@ func originContainedBy(candidate, container Origin) bool { return false } } + if (len(candidate.contributors) == 0) != (len(container.contributors) == 0) { + return false + } + for _, contributor := range candidate.contributors { + found := false + for _, available := range container.contributors { + if contributor.location == available.location && + slices.Equal(contributor.source.segments, available.source.segments) && + sameInheritanceTrace(contributor.trace, available.trace) { + found = true + break + } + } + if !found { + return false + } + } return true } diff --git a/internal/engine/records.go b/internal/engine/records.go index a2df796..0f19257 100644 --- a/internal/engine/records.go +++ b/internal/engine/records.go @@ -201,13 +201,25 @@ type Diagnostic struct { Related []Location } +// InheritanceStep identifies a concrete payload transfer controlled by a +// relationship directive. Locations belong to that directive, not its value. +type InheritanceStep struct { + From Path + To Path + Reference Path + Locations []Location +} + // Origin describes where an effective definition came from. Multiple locations // represent a composite origin; resolution origins are synthetic and identify their -// operation instead of a file location. +// operation instead of a file location. Inheritance is projected from private +// contributor traces at inspection boundaries. type Origin struct { - Layer Layer - Locations []Location - Operation ChangeKind + Layer Layer + Locations []Location + Operation ChangeKind + Inheritance []InheritanceStep + contributors []inheritanceContributor } // FieldInfo describes an effective configuration path. @@ -261,9 +273,32 @@ func cloneSourceInfos(sources []SourceInfo) []SourceInfo { func cloneOrigin(origin Origin) Origin { origin.Locations = cloneLocations(origin.Locations) + origin.Inheritance = cloneInheritanceSteps(origin.Inheritance) + if origin.contributors != nil { + contributors := make([]inheritanceContributor, len(origin.contributors)) + for index, contributor := range origin.contributors { + contributors[index] = contributor + contributors[index].source = contributor.source.clone() + } + origin.contributors = contributors + } return origin } +func cloneInheritanceSteps(steps []InheritanceStep) []InheritanceStep { + if steps == nil { + return nil + } + cloned := make([]InheritanceStep, len(steps)) + for index, step := range steps { + cloned[index] = InheritanceStep{ + From: step.From.clone(), To: step.To.clone(), Reference: step.Reference.clone(), + Locations: cloneLocations(step.Locations), + } + } + return cloned +} + func cloneFieldInfo(info FieldInfo) FieldInfo { info.Origin = cloneOrigin(info.Origin) return info