Keep sequence projection storage proportional to assigned items

docs/roadmap/remediation.md

@@ -783,7 +783,15 @@ scope; do not change selector meaning or require consumers to remove valid rules
 
 ## 19. Avoid Dense Global-Position Storage For Sparse Item Projections — P2
 
-Status: [ ] Open.
+Status: [x] Complete.
+
+Completion: contiguous composition ranges and their origin contributions retain
+arithmetic offset lookup. Scattered projections use independently cloned sparse
+maps in both directions; unprepared range queries no longer allocate indexes.
+Audited the shared projection/history callers for batch preparation. Added
+structural coverage for many one-item ranges, widely separated assignments, and
+absent coordinates. Existing upsert, history, version, and ownership tests pass,
+along with `./scripts/check.sh` and `go test -race ./internal/engine`.
 
 Owners: `newSequenceItemProjectionIndex` in
 [`sequence_projection.go`](../../internal/engine/sequence_projection.go),

internal/engine/compose.go

@@ -1,6 +1,10 @@
 package engine
 
-import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+import (
+	"maps"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
 
 // composedDocument is the private handoff between source assembly and later
 // precedence layers. Claims remain ordered by source and declaration traversal;
@@ -89,19 +93,18 @@ type sequenceRange struct {
 	projection  sequenceItemProjectionIndex
 }
 
-func (assigned sequenceRange) projections() []sequenceItemProjection {
-	if assigned.items != nil {
-		return append([]sequenceItemProjection{}, assigned.items...)
-	}
-	return sequenceItemProjections(assigned.end-assigned.start, assigned.start)
-}
-
 func (assigned sequenceRange) outputForInput(inputIndex int) (int, bool) {
 	if assigned.projection.outputByInput != nil {
 		return assigned.projection.outputForInput(inputIndex)
 	}
 	if assigned.items != nil {
-		return newSequenceItemProjectionIndex(assigned.items).outputForInput(inputIndex)
+		output, found := 0, false
+		for _, item := range assigned.items {
+			if item.inputIndex == inputIndex && inputIndex >= 0 && item.outputIndex >= 0 {
+				output, found = item.outputIndex, true
+			}
+		}
+		return output, found
 	}
 	if inputIndex < 0 || inputIndex >= assigned.end-assigned.start {
 		return 0, false
@@ -114,7 +117,13 @@ func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) {
 		return assigned.projection.inputForOutput(outputIndex)
 	}
 	if assigned.items != nil {
-		return newSequenceItemProjectionIndex(assigned.items).inputForOutput(outputIndex)
+		input, found := 0, false
+		for _, item := range assigned.items {
+			if item.outputIndex == outputIndex && outputIndex >= 0 && item.inputIndex >= 0 {
+				input, found = item.inputIndex, true
+			}
+		}
+		return input, found
 	}
 	if outputIndex < assigned.start || outputIndex >= assigned.end {
 		return 0, false
@@ -123,8 +132,8 @@ func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) {
 }
 
 func indexSequenceRange(assigned sequenceRange) sequenceRange {
-	if assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil {
+	if assigned.items != nil && assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil {
-		assigned.projection = newSequenceItemProjectionIndex(assigned.projections())
+		assigned.projection = newSequenceItemProjectionIndex(assigned.items)
 	}
 	return assigned
 }
@@ -459,8 +468,8 @@ func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange {
 		if assigned.items != nil {
 			cloned[i].items = append([]sequenceItemProjection{}, assigned.items...)
 		}
-		cloned[i].projection.outputByInput = append([]int(nil), assigned.projection.outputByInput...)
+		cloned[i].projection.outputByInput = maps.Clone(assigned.projection.outputByInput)
-		cloned[i].projection.inputByOutput = append([]int(nil), assigned.projection.inputByOutput...)
+		cloned[i].projection.inputByOutput = maps.Clone(assigned.projection.inputByOutput)
 	}
 	return cloned
 }
@@ -502,9 +511,15 @@ func projectedCompositionOrigins(inputs []*document.Node, ranges []sequenceRange
 	origins := make(map[string]Origin)
 	sequencesBySource := make([][]sequenceHistoryContribution, len(inputs))
 	for _, assigned := range ranges {
-		sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], sequenceHistoryContribution{
+		contribution := sequenceHistoryContribution{
-			path: assigned.path.clone(), mode: sequenceHistoryMapped, items: assigned.projections(),
+			path: assigned.path.clone(), mode: sequenceHistoryAppend,
-		})
+			offset: assigned.start, length: assigned.end - assigned.start,
+		}
+		if assigned.items != nil {
+			contribution.mode = sequenceHistoryMapped
+			contribution.items = assigned.items
+		}
+		sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], contribution)
 	}
 	for sourceIndex, input := range inputs {
 		application := historyApplication{value: input, sequences: sequencesBySource[sourceIndex]}

internal/engine/sequence_projection.go

@@ -1,33 +1,17 @@
 package engine
 
 type sequenceItemProjectionIndex struct {
-	outputByInput []int
+	outputByInput map[int]int
-	inputByOutput []int
+	inputByOutput map[int]int
 }
 
 func newSequenceItemProjectionIndex(projections []sequenceItemProjection) sequenceItemProjectionIndex {
-	maxInput := -1
-	maxOutput := -1
-	for _, projection := range projections {
-		if projection.inputIndex > maxInput {
-			maxInput = projection.inputIndex
-		}
-		if projection.outputIndex > maxOutput {
-			maxOutput = projection.outputIndex
-		}
-	}
 	index := sequenceItemProjectionIndex{
-		outputByInput: make([]int, maxInput+1),
+		outputByInput: make(map[int]int, len(projections)),
-		inputByOutput: make([]int, maxOutput+1),
+		inputByOutput: make(map[int]int, len(projections)),
-	}
-	for input := range index.outputByInput {
-		index.outputByInput[input] = -1
-	}
-	for output := range index.inputByOutput {
-		index.inputByOutput[output] = -1
 	}
 	for _, projection := range projections {
-		if projection.inputIndex >= 0 && projection.inputIndex < len(index.outputByInput) && projection.outputIndex >= 0 && projection.outputIndex < len(index.inputByOutput) {
+		if projection.inputIndex >= 0 && projection.outputIndex >= 0 {
 			index.outputByInput[projection.inputIndex] = projection.outputIndex
 			index.inputByOutput[projection.outputIndex] = projection.inputIndex
 		}
@@ -36,15 +20,11 @@ func newSequenceItemProjectionIndex(projections []sequenceItemProjection) sequen
 }
 
 func (index sequenceItemProjectionIndex) outputForInput(inputIndex int) (int, bool) {
-	if inputIndex < 0 || inputIndex >= len(index.outputByInput) || index.outputByInput[inputIndex] < 0 {
+	output, found := index.outputByInput[inputIndex]
-		return 0, false
+	return output, found
-	}
-	return index.outputByInput[inputIndex], true
 }
 
 func (index sequenceItemProjectionIndex) inputForOutput(outputIndex int) (int, bool) {
-	if outputIndex < 0 || outputIndex >= len(index.inputByOutput) || index.inputByOutput[outputIndex] < 0 {
+	input, found := index.inputByOutput[outputIndex]
-		return 0, false
+	return input, found
-	}
-	return index.inputByOutput[outputIndex], true
 }

internal/engine/sequence_projection_test.go

@@ -0,0 +1,49 @@
+package engine
+
+import "testing"
+
+func TestSparseSequenceProjectionIndexesOnlyAssignedPositions(t *testing.T) {
+	index := newSequenceItemProjectionIndex([]sequenceItemProjection{
+		{inputIndex: 0, outputIndex: 9000}, {inputIndex: 3000, outputIndex: 2},
+	})
+	if len(index.outputByInput) != 2 || len(index.inputByOutput) != 2 {
+		t.Fatal("sparse projection retained unassigned positions")
+	}
+	for input, output := range map[int]int{0: 9000, 3000: 2} {
+		if got, found := index.outputForInput(input); !found || got != output {
+			t.Fatalf("input %d = %d, %t", input, got, found)
+		}
+		if got, found := index.inputForOutput(output); !found || got != input {
+			t.Fatalf("output %d = %d, %t", output, got, found)
+		}
+	}
+	for _, position := range []int{-1, 1, 10000} {
+		if _, found := index.outputForInput(position); found {
+			t.Fatalf("unassigned input %d exists", position)
+		}
+		if _, found := index.inputForOutput(position); found {
+			t.Fatalf("unassigned output %d exists", position)
+		}
+	}
+}
+
+func TestContiguousSequenceRangesKeepArithmeticLookup(t *testing.T) {
+	ranges := make([]sequenceRange, 128)
+	for position := range ranges {
+		ranges[position] = sequenceRange{sourceIndex: position, path: Keys("items"), start: position, end: position + 1}
+	}
+	for _, assigned := range indexSequenceRanges(ranges) {
+		if len(assigned.projection.outputByInput) != 0 || len(assigned.projection.inputByOutput) != 0 {
+			t.Fatal("contiguous range materialized a projection index")
+		}
+		if output, found := assigned.outputForInput(0); !found || output != assigned.start {
+			t.Fatal("arithmetic forward lookup lost its offset")
+		}
+		if input, found := assigned.inputForOutput(assigned.start); !found || input != 0 {
+			t.Fatal("arithmetic reverse lookup lost its offset")
+		}
+		if _, found := assigned.inputForOutput(assigned.end); found {
+			t.Fatal("range includes its exclusive end")
+		}
+	}
+}