From d0848290daf658919d755e1938207f0dd3f39e8d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 24 Oct 2026 14:04:26 +0000 Subject: [PATCH] Keep sequence projection storage proportional to assigned items --- docs/roadmap/remediation.md | 10 ++++- internal/engine/compose.go | 49 ++++++++++++++------- internal/engine/sequence_projection.go | 38 ++++------------ internal/engine/sequence_projection_test.go | 49 +++++++++++++++++++++ 4 files changed, 99 insertions(+), 47 deletions(-) create mode 100644 internal/engine/sequence_projection_test.go diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 7efeeba..c363452 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -783,7 +783,15 @@ scope; do not change selector meaning or require consumers to remove valid rules ## 19. Avoid Dense Global-Position Storage For Sparse Item Projections — P2 -Status: [ ] Open. +Status: [x] Complete. + +Completion: contiguous composition ranges and their origin contributions retain +arithmetic offset lookup. Scattered projections use independently cloned sparse +maps in both directions; unprepared range queries no longer allocate indexes. +Audited the shared projection/history callers for batch preparation. Added +structural coverage for many one-item ranges, widely separated assignments, and +absent coordinates. Existing upsert, history, version, and ownership tests pass, +along with `./scripts/check.sh` and `go test -race ./internal/engine`. Owners: `newSequenceItemProjectionIndex` in [`sequence_projection.go`](../../internal/engine/sequence_projection.go), diff --git a/internal/engine/compose.go b/internal/engine/compose.go index 3db2435..efb26ea 100644 --- a/internal/engine/compose.go +++ b/internal/engine/compose.go @@ -1,6 +1,10 @@ package engine -import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +import ( + "maps" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) // composedDocument is the private handoff between source assembly and later // precedence layers. Claims remain ordered by source and declaration traversal; @@ -89,19 +93,18 @@ type sequenceRange struct { projection sequenceItemProjectionIndex } -func (assigned sequenceRange) projections() []sequenceItemProjection { - if assigned.items != nil { - return append([]sequenceItemProjection{}, assigned.items...) - } - return sequenceItemProjections(assigned.end-assigned.start, assigned.start) -} - func (assigned sequenceRange) outputForInput(inputIndex int) (int, bool) { if assigned.projection.outputByInput != nil { return assigned.projection.outputForInput(inputIndex) } if assigned.items != nil { - return newSequenceItemProjectionIndex(assigned.items).outputForInput(inputIndex) + output, found := 0, false + for _, item := range assigned.items { + if item.inputIndex == inputIndex && inputIndex >= 0 && item.outputIndex >= 0 { + output, found = item.outputIndex, true + } + } + return output, found } if inputIndex < 0 || inputIndex >= assigned.end-assigned.start { return 0, false @@ -114,7 +117,13 @@ func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) { return assigned.projection.inputForOutput(outputIndex) } if assigned.items != nil { - return newSequenceItemProjectionIndex(assigned.items).inputForOutput(outputIndex) + input, found := 0, false + for _, item := range assigned.items { + if item.outputIndex == outputIndex && outputIndex >= 0 && item.inputIndex >= 0 { + input, found = item.inputIndex, true + } + } + return input, found } if outputIndex < assigned.start || outputIndex >= assigned.end { return 0, false @@ -123,8 +132,8 @@ func (assigned sequenceRange) inputForOutput(outputIndex int) (int, bool) { } func indexSequenceRange(assigned sequenceRange) sequenceRange { - if assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil { - assigned.projection = newSequenceItemProjectionIndex(assigned.projections()) + if assigned.items != nil && assigned.projection.outputByInput == nil && assigned.projection.inputByOutput == nil { + assigned.projection = newSequenceItemProjectionIndex(assigned.items) } return assigned } @@ -459,8 +468,8 @@ func cloneSequenceRanges(ranges []sequenceRange) []sequenceRange { if assigned.items != nil { cloned[i].items = append([]sequenceItemProjection{}, assigned.items...) } - cloned[i].projection.outputByInput = append([]int(nil), assigned.projection.outputByInput...) - cloned[i].projection.inputByOutput = append([]int(nil), assigned.projection.inputByOutput...) + cloned[i].projection.outputByInput = maps.Clone(assigned.projection.outputByInput) + cloned[i].projection.inputByOutput = maps.Clone(assigned.projection.inputByOutput) } return cloned } @@ -502,9 +511,15 @@ func projectedCompositionOrigins(inputs []*document.Node, ranges []sequenceRange origins := make(map[string]Origin) sequencesBySource := make([][]sequenceHistoryContribution, len(inputs)) for _, assigned := range ranges { - sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], sequenceHistoryContribution{ - path: assigned.path.clone(), mode: sequenceHistoryMapped, items: assigned.projections(), - }) + contribution := sequenceHistoryContribution{ + path: assigned.path.clone(), mode: sequenceHistoryAppend, + offset: assigned.start, length: assigned.end - assigned.start, + } + if assigned.items != nil { + contribution.mode = sequenceHistoryMapped + contribution.items = assigned.items + } + sequencesBySource[assigned.sourceIndex] = append(sequencesBySource[assigned.sourceIndex], contribution) } for sourceIndex, input := range inputs { application := historyApplication{value: input, sequences: sequencesBySource[sourceIndex]} diff --git a/internal/engine/sequence_projection.go b/internal/engine/sequence_projection.go index ecb6827..0890a99 100644 --- a/internal/engine/sequence_projection.go +++ b/internal/engine/sequence_projection.go @@ -1,33 +1,17 @@ package engine type sequenceItemProjectionIndex struct { - outputByInput []int - inputByOutput []int + outputByInput map[int]int + inputByOutput map[int]int } func newSequenceItemProjectionIndex(projections []sequenceItemProjection) sequenceItemProjectionIndex { - maxInput := -1 - maxOutput := -1 - for _, projection := range projections { - if projection.inputIndex > maxInput { - maxInput = projection.inputIndex - } - if projection.outputIndex > maxOutput { - maxOutput = projection.outputIndex - } - } index := sequenceItemProjectionIndex{ - outputByInput: make([]int, maxInput+1), - inputByOutput: make([]int, maxOutput+1), - } - for input := range index.outputByInput { - index.outputByInput[input] = -1 - } - for output := range index.inputByOutput { - index.inputByOutput[output] = -1 + outputByInput: make(map[int]int, len(projections)), + inputByOutput: make(map[int]int, len(projections)), } for _, projection := range projections { - if projection.inputIndex >= 0 && projection.inputIndex < len(index.outputByInput) && projection.outputIndex >= 0 && projection.outputIndex < len(index.inputByOutput) { + if projection.inputIndex >= 0 && projection.outputIndex >= 0 { index.outputByInput[projection.inputIndex] = projection.outputIndex index.inputByOutput[projection.outputIndex] = projection.inputIndex } @@ -36,15 +20,11 @@ func newSequenceItemProjectionIndex(projections []sequenceItemProjection) sequen } func (index sequenceItemProjectionIndex) outputForInput(inputIndex int) (int, bool) { - if inputIndex < 0 || inputIndex >= len(index.outputByInput) || index.outputByInput[inputIndex] < 0 { - return 0, false - } - return index.outputByInput[inputIndex], true + output, found := index.outputByInput[inputIndex] + return output, found } func (index sequenceItemProjectionIndex) inputForOutput(outputIndex int) (int, bool) { - if outputIndex < 0 || outputIndex >= len(index.inputByOutput) || index.inputByOutput[outputIndex] < 0 { - return 0, false - } - return index.inputByOutput[outputIndex], true + input, found := index.inputByOutput[outputIndex] + return input, found } diff --git a/internal/engine/sequence_projection_test.go b/internal/engine/sequence_projection_test.go new file mode 100644 index 0000000..5d8738c --- /dev/null +++ b/internal/engine/sequence_projection_test.go @@ -0,0 +1,49 @@ +package engine + +import "testing" + +func TestSparseSequenceProjectionIndexesOnlyAssignedPositions(t *testing.T) { + index := newSequenceItemProjectionIndex([]sequenceItemProjection{ + {inputIndex: 0, outputIndex: 9000}, {inputIndex: 3000, outputIndex: 2}, + }) + if len(index.outputByInput) != 2 || len(index.inputByOutput) != 2 { + t.Fatal("sparse projection retained unassigned positions") + } + for input, output := range map[int]int{0: 9000, 3000: 2} { + if got, found := index.outputForInput(input); !found || got != output { + t.Fatalf("input %d = %d, %t", input, got, found) + } + if got, found := index.inputForOutput(output); !found || got != input { + t.Fatalf("output %d = %d, %t", output, got, found) + } + } + for _, position := range []int{-1, 1, 10000} { + if _, found := index.outputForInput(position); found { + t.Fatalf("unassigned input %d exists", position) + } + if _, found := index.inputForOutput(position); found { + t.Fatalf("unassigned output %d exists", position) + } + } +} + +func TestContiguousSequenceRangesKeepArithmeticLookup(t *testing.T) { + ranges := make([]sequenceRange, 128) + for position := range ranges { + ranges[position] = sequenceRange{sourceIndex: position, path: Keys("items"), start: position, end: position + 1} + } + for _, assigned := range indexSequenceRanges(ranges) { + if len(assigned.projection.outputByInput) != 0 || len(assigned.projection.inputByOutput) != 0 { + t.Fatal("contiguous range materialized a projection index") + } + if output, found := assigned.outputForInput(0); !found || output != assigned.start { + t.Fatal("arithmetic forward lookup lost its offset") + } + if input, found := assigned.inputForOutput(assigned.start); !found || input != 0 { + t.Fatal("arithmetic reverse lookup lost its offset") + } + if _, found := assigned.inputForOutput(assigned.end); found { + t.Fatal("range includes its exclusive end") + } + } +}