Index mapping assembly for composition and overlays

docs/roadmap/implementation.md

@@ -220,6 +220,15 @@ filesystem snapshot, universal sandbox, or transaction guarantee is added.
 
 ## Stage 1: Index mapping assembly and establish scaling baselines
 
+**Completed 2026-09-29.** Private per-assembly mapping indexes now serve strict
+composition and recursive overlay lookup while ordered entries and diagnostics
+remain stable; conflict groups are checked once. Focused Compose/Overlay tests
+and the everyday checker passed. On Linux amd64, the 512-entry benchmark
+fixtures measured approximately 4.0 ms/op before and 3.3 ms/op after for wide
+composition, and 12.9 ms/op before and 12.0 ms/op after for recursive overlay.
+These are comparative evidence, not timing gates; provenance work still
+dominates the latter and belongs to the next stage.
+
 Scope: `internal/document` and engine composition/overlay lookup, with existing
 behavior only.
 

internal/engine/assembly_bench_test.go

@@ -0,0 +1,57 @@
+package engine
+
+import (
+	"strconv"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func BenchmarkComposeWideMappings(b *testing.B) {
+	for _, width := range []int{128, 256, 512} {
+		b.Run(strconv.Itoa(width), func(b *testing.B) {
+			left := benchmarkCatalog("source-1", 0, width)
+			right := benchmarkCatalog("source-2", width, width)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for range b.N {
+				if _, diagnostics := composeDocuments(left, right); len(diagnostics) != 0 {
+					b.Fatal(diagnostics)
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkOverlayRecursiveMappings(b *testing.B) {
+	for _, width := range []int{128, 256, 512} {
+		b.Run(strconv.Itoa(width), func(b *testing.B) {
+			base, diagnostics := composeDocuments(benchmarkCatalog("source-1", 0, width))
+			if len(diagnostics) != 0 {
+				b.Fatal(diagnostics)
+			}
+			overlay := benchmarkCatalog("source-2", width/2, width)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for range b.N {
+				if _, diagnostics := overlayDocuments(base, overlay); len(diagnostics) != 0 {
+					b.Fatal(diagnostics)
+				}
+			}
+		})
+	}
+}
+
+func benchmarkCatalog(source string, first, width int) *document.Node {
+	location := document.Location{SourceID: source}
+	entries := make([]document.Entry, width)
+	for index := range entries {
+		entries[index] = document.Entry{
+			Key:   "item-" + strconv.Itoa(first+index),
+			Value: &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{{Key: "enabled", Value: &document.Node{Kind: document.Boolean, Text: "true", Location: location}}}},
+		}
+	}
+	return &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{{
+		Key: "catalog", Value: &document.Node{Kind: document.Mapping, Location: location, Mapping: entries},
+	}}}
+}

internal/engine/compose.go

@@ -58,8 +58,14 @@ func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic
 		collected.append(input, Path{})
 	}
 
+	checked := make(map[string]struct{}, len(collected.byPath))
 	for _, claim := range collected.ordered {
-		claims := collected.byPath[claim.path.String()]
+		pointer := claim.path.String()
+		if _, seen := checked[pointer]; seen {
+			continue
+		}
+		checked[pointer] = struct{}{}
+		claims := collected.byPath[pointer]
 		if len(claims) < 2 || allMappingClaims(claims) {
 			continue
 		}
@@ -67,8 +73,9 @@ func composeDocuments(inputs ...*document.Node) (*composedDocument, []Diagnostic
 	}
 
 	root := document.Clone(inputs[0])
+	index := new(mappingIndex)
 	for _, input := range inputs[1:] {
-		mergeDisjointMappings(root, input)
+		mergeDisjointMappings(root, input, index)
 	}
 	applications := make([]historyApplication, 0, len(inputs))
 	for _, input := range inputs {
@@ -109,18 +116,18 @@ func allMappingClaims(claims []compositionClaim) bool {
 	return true
 }
 
-func mergeDisjointMappings(target, incoming *document.Node) {
+func mergeDisjointMappings(target, incoming *document.Node, index *mappingIndex) {
 	for _, incomingEntry := range incoming.Mapping {
-		index := documentEntryIndex(target.Mapping, incomingEntry.Key)
+		position := index.find(target, incomingEntry.Key)
-		if index < 0 {
+		if position < 0 {
 			cloned := incomingEntry
 			cloned.Value = document.Clone(incomingEntry.Value)
-			target.Mapping = append(target.Mapping, cloned)
+			index.append(target, cloned)
 			continue
 		}
 		// Claim validation guarantees that the only remaining overlap is a
 		// mapping pair. Empty mappings naturally contribute no child entries.
-		mergeDisjointMappings(target.Mapping[index].Value, incomingEntry.Value)
+		mergeDisjointMappings(target.Mapping[position].Value, incomingEntry.Value, index)
 	}
 }
 

internal/engine/compose_test.go

@@ -116,6 +116,31 @@ func TestComposeDocumentsReportsAllClaimantsAtExactLocations(t *testing.T) {
 	assertCompositionConflict(t, diagnostics, "/group", []string{"source-1", "source-2", "source-3"})
 }
 
+func TestComposeDocumentsKeepsFirstConflictAndAppendedEntryOrder(t *testing.T) {
+	first := parseCompositionDocument(t, "source-1", "catalog:\n  first: 1\n  shared: one\n  later: one\n")
+	second := parseCompositionDocument(t, "source-2", "catalog:\n  second: 2\n  shared: two\n  later: two\n")
+	third := parseCompositionDocument(t, "source-3", "catalog:\n  third: 3\n  shared: three\n")
+	_, diagnostics := composeDocuments(first, second, third)
+	assertCompositionConflict(t, diagnostics, "/catalog/shared", []string{"source-1", "source-2", "source-3"})
+
+	composed, diagnostics := composeDocuments(
+		parseCompositionDocument(t, "source-1", "catalog:\n  first: 1\n"),
+		parseCompositionDocument(t, "source-2", "catalog:\n  second: 2\n"),
+		parseCompositionDocument(t, "source-3", "catalog:\n  third: 3\n"),
+	)
+	if len(diagnostics) != 0 {
+		t.Fatalf("composeDocuments() diagnostics = %#v", diagnostics)
+	}
+	entries := composed.root.Mapping[0].Value.Mapping
+	got := make([]string, len(entries))
+	for index, entry := range entries {
+		got[index] = entry.Key
+	}
+	if want := []string{"first", "second", "third"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("catalog keys = %#v, want %#v", got, want)
+	}
+}
+
 func TestComposeDocumentsTreatsEmptyMappingsAsChildlessContributions(t *testing.T) {
 	empty, diagnostics := composeDocuments(
 		parseCompositionDocument(t, "source-1", "{}\n"),

internal/engine/mapping_index.go

@@ -0,0 +1,37 @@
+package engine
+
+import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+
+// mappingIndex keeps ordered document entries and their lookup positions in
+// sync while one composition or overlay is assembled.
+type mappingIndex struct {
+	byNode map[*document.Node]map[string]int
+}
+
+func (m *mappingIndex) entries(node *document.Node) map[string]int {
+	if m.byNode == nil {
+		m.byNode = make(map[*document.Node]map[string]int)
+	}
+	if entries, ok := m.byNode[node]; ok {
+		return entries
+	}
+	entries := make(map[string]int, len(node.Mapping))
+	for index, entry := range node.Mapping {
+		entries[entry.Key] = index
+	}
+	m.byNode[node] = entries
+	return entries
+}
+
+func (m *mappingIndex) find(node *document.Node, key string) int {
+	if index, ok := m.entries(node)[key]; ok {
+		return index
+	}
+	return -1
+}
+
+func (m *mappingIndex) append(node *document.Node, entry document.Entry) {
+	entries := m.entries(node)
+	entries[entry.Key] = len(node.Mapping)
+	node.Mapping = append(node.Mapping, entry)
+}

internal/engine/overlay.go

@@ -32,29 +32,33 @@ func overlayDocuments(base *composedDocument, overlay *document.Node) (*composed
 }
 
 func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool) {
+	mergeOverlayMappingsIndexed(result, target, overlay, current, layer, recordReplacements, new(mappingIndex))
+}
+
+func mergeOverlayMappingsIndexed(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool, index *mappingIndex) {
 	for _, incoming := range overlay.Mapping {
 		path := current.Key(incoming.Key)
-		index := documentEntryIndex(target.Mapping, incoming.Key)
+		position := index.find(target, incoming.Key)
-		if index < 0 {
+		if position < 0 {
 			cloned := incoming
 			cloned.Value = document.Clone(incoming.Value)
-			target.Mapping = append(target.Mapping, cloned)
+			index.append(target, cloned)
 			collectOverlayOrigins(result.origins, incoming.Value, path, layer)
 			continue
 		}
 
-		existing := target.Mapping[index].Value
+		existing := target.Mapping[position].Value
 		if existing.Kind == document.Mapping && incoming.Value.Kind == document.Mapping {
 			mergeOverlayOrigin(result.origins, path, incoming.Value.Location, layer)
-			mergeOverlayMappings(result, existing, incoming.Value, path, layer, recordReplacements)
+			mergeOverlayMappingsIndexed(result, existing, incoming.Value, path, layer, recordReplacements, index)
 			continue
 		}
 
 		displaced := displacedOrigins(existing, result.origins, path)
 		previous := cloneOrigin(result.origins[path.String()])
 		deleteOriginSubtree(result.origins, path)
-		target.Mapping[index] = incoming
+		target.Mapping[position] = incoming
-		target.Mapping[index].Value = document.Clone(incoming.Value)
+		target.Mapping[position].Value = document.Clone(incoming.Value)
 		collectOverlayOrigins(result.origins, incoming.Value, path, layer)
 		if recordReplacements {
 			result.overlays = append(result.overlays, overlayRecord{