Index sequence assembly and history lookups

docs/roadmap/remediation.md

@@ -222,7 +222,15 @@ mapping is still explicit. Defaults-only overrides keep their existing behavior.
 
 ## 7. Remove new quadratic scans from sequence construction — P2
 
-- [ ] Addressed and behaviorally validated.
+- [x] Addressed and behaviorally validated.
+
+Completion: claims retain encountered immutable nodes; history construction
+groups ranges by source; scope/version projection uses source/path range
+lookups; retained-history traversal and replay use per-application path indexes.
+Ordered slices and detached clone ownership remain intact. Static review confirms
+the listed nested scans are removed. A wide-collection replay/origin/ownership
+regression, the everyday check, and `go test -race ./...` passed. No timing
+threshold or release gate was added.
 
 **Owners:** [compose.go](../../internal/engine/compose.go), especially
 `validateRegisteredSequences` and application construction; range lookups in

internal/engine/compose.go

@@ -59,6 +59,7 @@ func cloneComposedDocument(base *composedDocument) *composedDocument {
 }
 
 type compositionClaim struct {
+	node        *document.Node // borrowed immutable contribution, not the assembly tree
 	path        Path
 	kind        document.Kind
 	location    Location
@@ -121,7 +122,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 		return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)}
 	}
 
-	validated, ranges, diagnostics := validateRegisteredSequences(inputs, collected.ordered, policy)
+	validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy)
 	if len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
@@ -133,12 +134,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 	}
 	applications := make([]historyApplication, 0, len(inputs))
 	seenSequences := make(map[string]bool)
+	rangesBySource := make([][]sequenceRange, len(inputs))
+	for _, assigned := range ranges {
+		rangesBySource[assigned.sourceIndex] = append(rangesBySource[assigned.sourceIndex], assigned)
+	}
 	for sourceIndex, input := range inputs {
 		application := historyApplication{layer: LayerBase, value: input}
-		for _, assigned := range ranges {
+		for _, assigned := range rangesBySource[sourceIndex] {
-			if assigned.sourceIndex != sourceIndex {
-				continue
-			}
 			pointer := assigned.path.String()
 			mode := sequenceHistoryAppend
 			if !seenSequences[pointer] {
@@ -149,6 +151,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
 				path: assigned.path.clone(), mode: mode, offset: assigned.start, length: assigned.end - assigned.start,
 			})
 		}
+		application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
 		applications = append(applications, application)
 	}
 	origins := projectedCompositionOrigins(inputs, ranges)
@@ -183,6 +186,7 @@ func completeCompositionOrigins(node *document.Node, path Path, origins map[stri
 
 func (c *compositionClaims) append(node *document.Node, path Path, sourceIndex int) {
 	claim := compositionClaim{
+		node:        node,
 		path:        path.clone(),
 		kind:        node.Kind,
 		location:    publicLocation(node.Location),
@@ -213,7 +217,7 @@ type validatedSequence struct {
 	items []*document.Node
 }
 
-func validateRegisteredSequences(inputs []*document.Node, claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
+func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
 	itemsByPath := make(map[string]map[int]validatedSequence)
 	identitiesByPath := make(map[string]map[string]Location)
 	lengthsByPath := make(map[string]int)
@@ -226,10 +230,7 @@ func validateRegisteredSequences(inputs []*document.Node, claims []compositionCl
 		if !registered {
 			continue
 		}
-		node, ok := documentAtPath(inputs[claim.sourceIndex], claim.path)
+		node := claim.node
-		if !ok || node.Kind != document.Sequence {
-			continue
-		}
 		pathKey := claim.path.String()
 		identities := identitiesByPath[pathKey]
 		if identities == nil {
@@ -389,6 +390,19 @@ type sequenceRangeKey struct {
 	path        string
 }
 
+type sequenceRangeIndex map[sequenceRangeKey]sequenceRange
+
+func indexSequenceRanges(ranges []sequenceRange) sequenceRangeIndex {
+	if len(ranges) == 0 {
+		return nil
+	}
+	index := make(sequenceRangeIndex, len(ranges))
+	for _, assigned := range ranges {
+		index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned
+	}
+	return index
+}
+
 // Keep local paths for matching Stage 2 ranges and effective paths for emitted
 // evidence. The paths are transformed segment by segment; pointer strings are
 // only used as map keys after projection is complete.

internal/engine/compose_test.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -8,6 +9,52 @@ import (
 	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
 )
 
+func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) {
+	policy := sequencePolicy{identityFields: make(map[string]string)}
+	var inputs []*document.Node
+	for source := 0; source < 3; source++ {
+		var text strings.Builder
+		for collection := 0; collection < 128; collection++ {
+			key := fmt.Sprintf("records%d", collection)
+			policy.identityFields[Keys(key).String()] = "id"
+			fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source)
+		}
+		inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String()))
+	}
+	composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	working := &document.Node{Kind: document.Mapping}
+	for _, application := range composed.applications {
+		applyHistoryApplication(&working, application, new(mappingIndex))
+	}
+	if !configurationValuesEqual(working, composed.root) {
+		t.Fatal("wide registered collections replay differs from assembly")
+	}
+	for collection := 0; collection < 128; collection++ {
+		path := Keys(fmt.Sprintf("records%d", collection)).Index(2).Key("id")
+		origin := composed.origins[path.String()]
+		if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-2" {
+			t.Fatalf("shifted origin at %s = %#v", path.String(), origin)
+		}
+		local, ok := historyApplicationLocalPath(composed.applications[2], path)
+		if !ok || local.String() != Keys(fmt.Sprintf("records%d", collection)).Index(0).Key("id").String() {
+			t.Fatal("wide local path projection failed")
+		}
+	}
+	clone := cloneComposedDocument(composed)
+	for _, original := range composed.applications[0].sequences {
+		pointer := original.path.String()
+		copy := clone.applications[0].sequencesByPath[pointer]
+		copy.path.segments[0].key = "changed"
+		if composed.applications[0].sequencesByPath[pointer].path.String() != pointer {
+			t.Fatal("history path index aliases cloned state")
+		}
+		break
+	}
+}
+
 func TestComposeDocumentsJoinsDisjointMappingsAndTracksClaims(t *testing.T) {
 	base := parseCompositionDocument(t, "source-1", `services:
   api:

internal/engine/directives.go

@@ -167,6 +167,10 @@ func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node
 }
 
 func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourceIndex int, ranges []sequenceRange) *document.Node {
+	return fragmentVersionClaimWithIndex(root, rule, sourceIndex, indexSequenceRanges(ranges))
+}
+
+func fragmentVersionClaimWithIndex(root *document.Node, rule *VersionRule, sourceIndex int, ranges sequenceRangeIndex) *document.Node {
 	if rule == nil {
 		return nil
 	}
@@ -192,15 +196,11 @@ func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourc
 			if !valid {
 				return current
 			}
-			for _, assigned := range ranges {
+			if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found {
-				if assigned.sourceIndex != sourceIndex || assigned.path.String() != currentPath.String() {
-					continue
-				}
 				if index < assigned.start || index >= assigned.end {
 					return nil
 				}
 				index -= assigned.start
-				break
 			}
 			if index < 0 || index >= len(current.Sequence) {
 				return nil

internal/engine/history.go

@@ -45,11 +45,12 @@ func (c *retainedNodeCounter) add(root *document.Node) bool {
 // Replay reconciles with canonical decoding before domain resolution so both
 // contracts agree on the effective result without retaining a tree per field.
 type historyApplication struct {
-	layer     Layer
+	layer           Layer
-	path      Path
+	path            Path
-	value     *document.Node
+	value           *document.Node
-	origin    Origin
+	origin          Origin
-	sequences []sequenceHistoryContribution
+	sequences       []sequenceHistoryContribution
+	sequencesByPath map[string]sequenceHistoryContribution
 }
 
 func cloneHistoryApplications(values []historyApplication) []historyApplication {
@@ -62,6 +63,7 @@ func cloneHistoryApplications(values []historyApplication) []historyApplication
 			layer: value.layer, path: value.path.clone(), value: value.value,
 			origin: cloneOrigin(value.origin), sequences: cloneSequenceHistoryContributions(value.sequences),
 		}
+		cloned[index].sequencesByPath = indexSequenceHistoryContributions(cloned[index].sequences)
 	}
 	return cloned
 }
@@ -86,7 +88,7 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef
 				walk(entry.Value, local.Key(entry.Key), effective.Key(entry.Key))
 			}
 		case document.Sequence:
-			contribution, projected := sequenceHistoryAt(application.sequences, local)
+			contribution, projected := sequenceHistoryAt(application.sequencesByPath, local)
 			for index, child := range node.Sequence {
 				effectiveIndex := index
 				if projected {
@@ -108,7 +110,7 @@ func historyApplicationLocalPath(application historyApplication, effective Path)
 	}
 	local := Path{}
 	for _, segment := range relative.segments {
-		if contribution, found := sequenceHistoryAt(application.sequences, local); found {
+		if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found {
 			index, valid := schemaIndex(segment)
 			if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length {
 				return Path{}, false
@@ -201,7 +203,7 @@ func pathsIntersect(left, right Path) bool {
 
 func applyHistoryApplication(root **document.Node, application historyApplication, index *mappingIndex) {
 	if application.path.String() == "" {
-		mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequences)
+		mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath)
 		return
 	}
 	mergeHistoryAtPath(root, application.path, application.value, index)
@@ -211,7 +213,7 @@ func mergeHistoryDocument(target **document.Node, incoming *document.Node, index
 	mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil)
 }
 
-func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences []sequenceHistoryContribution) {
+func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) {
 	if incoming == nil {
 		return
 	}
@@ -236,13 +238,20 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node,
 	}
 }
 
-func sequenceHistoryAt(contributions []sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
+func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
-	for _, contribution := range contributions {
+	contribution, found := contributions[path.String()]
-		if contribution.path.String() == path.String() {
+	return contribution, found
-			return contribution, true
+}
-		}
+
+func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution {
+	if len(values) == 0 {
+		return nil
 	}
-	return sequenceHistoryContribution{}, false
+	index := make(map[string]sequenceHistoryContribution, len(values))
+	for _, value := range values {
+		index[value.path.String()] = value
+	}
+	return index
 }
 
 func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) {

internal/engine/imports.go

@@ -564,11 +564,12 @@ func importedVersionClaim(documents []*document.Node, composed *composedDocument
 	if composed == nil || rule == nil {
 		return nil
 	}
+	ranges := indexSequenceRanges(composed.sequenceRanges)
 	for index, contribution := range documents {
 		if index == mainIndex {
 			continue
 		}
-		if node := fragmentVersionClaimProjected(contribution, rule, index, composed.sequenceRanges); node != nil {
+		if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil {
 			location := publicLocation(node.Location)
 			return &Diagnostic{
 				Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
@@ -595,18 +596,13 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
 	}
 	applications := composed.applications
 	projectedApplications := make([]historyApplication, 0, len(applications))
+	ranges := indexSequenceRanges(composed.sequenceRanges)
 	for sourceIndex, contribution := range scope.contributions {
 		if contribution.nested == nil {
 			projectedApplications = append(projectedApplications, applications[sourceIndex])
 			continue
 		}
-		start := 0
+		start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
-		for _, assigned := range composed.sequenceRanges {
-			if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
-				start = assigned.start
-				break
-			}
-		}
 		prefix := contribution.destination.Index(start)
 		if sourceIndex < len(applications) {
 			outer := applications[sourceIndex]
@@ -622,6 +618,7 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
 				projected := nested
 				projected.path = appendDestination(prefix, nested.path)
 				projected.sequences = cloneSequenceHistoryContributions(nested.sequences)
+				projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences)
 				projectedApplications = append(projectedApplications, projected)
 			}
 		}
@@ -656,19 +653,14 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
 	}
 	var projectScope func(*composedDocument, acquisitionScope, Path, bool, string)
 	projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) {
+		ranges := indexSequenceRanges(currentComposition.sequenceRanges)
 		for sourceIndex, contribution := range current.contributions {
 			if contribution.source.ID == parentSourceID {
 				continue
 			}
 			contributionPrefix := prefix
 			contributionInsideItem := insideItem
-			start := 0
+			start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
-			for _, assigned := range currentComposition.sequenceRanges {
-				if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
-					start = assigned.start
-					break
-				}
-			}
 			if contribution.source.placementMode == placementModeSequenceItem {
 				container := appendDestination(contributionPrefix, contribution.destination)
 				contribution.source.Destination = container.clone()
@@ -692,7 +684,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
 				}
 				projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex)
 			} else {
-				projectSourceVersionEvidence(contribution.processing, contribution.destination, currentComposition.sequenceRanges, sourceIndex)
+				projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex)
 			}
 			if contribution.nested != nil && contribution.nested.acquisition != nil {
 				for _, nestedScope := range contribution.nested.acquisition {
@@ -704,7 +696,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
 	projectScope(composed, scope, Path{}, false, "")
 }
 
-func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges []sequenceRange, sourceIndex int) {
+func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) {
 	if processing != nil && processing.evidence != nil {
 		evidence := processing.evidence
 		evidence.destination = destination.clone()
@@ -713,11 +705,12 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des
 		evidence.projectedPaths = nil
 		paths := evidence.sensitivityPaths()
 		for index, path := range paths {
-			for _, assigned := range ranges {
+			for position := range path.segments {
-				if assigned.sourceIndex != sourceIndex || !pathContains(assigned.path, path) || len(path.segments) <= len(assigned.path.segments) {
+				prefix := Path{segments: path.segments[:position]}
+				assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}]
+				if !found {
 					continue
 				}
-				position := len(assigned.path.segments)
 				item, ok := schemaIndex(path.segments[position])
 				if ok && path.segments[position].kind != pathSegmentKey {
 					path.segments[position] = pathSegment{kind: pathSegmentIndex, index: assigned.start + item}

internal/engine/load_request.go

@@ -196,8 +196,9 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
 		return nil, sources, contexts, diagnostics
 	}
 	if composition != nil && rule != nil {
+		ranges := indexSequenceRanges(composition.sequenceRanges)
 		for sourceIndex, document := range documents {
-			if node := fragmentVersionClaimProjected(document, rule, sourceIndex, composition.sequenceRanges); node != nil {
+			if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
 				location := publicLocation(node.Location)
 				return nil, sources, contexts, []Diagnostic{{
 					Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),