eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Index sequence assembly and history lookups

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-17 04:52:20 +00:00
parent 527a38daf1
commit 023e428b9c
7 changed files with 124 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/roadmap/remediation.md
View File

@@ -222,7 +222,15 @@ mapping is still explicit. Defaults-only overrides keep their existing behavior.
## 7. Remove new quadratic scans from sequence construction — P2
- [ ] Addressed and behaviorally validated.
- [x] Addressed and behaviorally validated.
Completion: claims retain encountered immutable nodes; history construction
groups ranges by source; scope/version projection uses source/path range
lookups; retained-history traversal and replay use per-application path indexes.
Ordered slices and detached clone ownership remain intact. Static review confirms
the listed nested scans are removed. A wide-collection replay/origin/ownership
regression, the everyday check, and `go test -race ./...` passed. No timing
threshold or release gate was added.
**Owners:** [compose.go](../../internal/engine/compose.go), especially
`validateRegisteredSequences` and application construction; range lookups in

34
internal/engine/compose.go
View File

@@ -59,6 +59,7 @@ func cloneComposedDocument(base *composedDocument) *composedDocument {
}
type compositionClaim struct {
node *document.Node // borrowed immutable contribution, not the assembly tree
path Path
kind document.Kind
location Location
@@ -121,7 +122,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)}
}
validated, ranges, diagnostics := validateRegisteredSequences(inputs, collected.ordered, policy)
validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy)
if len(diagnostics) != 0 {
return nil, diagnostics
}
@@ -133,12 +134,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
}
applications := make([]historyApplication, 0, len(inputs))
seenSequences := make(map[string]bool)
rangesBySource := make([][]sequenceRange, len(inputs))
for _, assigned := range ranges {
rangesBySource[assigned.sourceIndex] = append(rangesBySource[assigned.sourceIndex], assigned)
}
for sourceIndex, input := range inputs {
application := historyApplication{layer: LayerBase, value: input}
for _, assigned := range ranges {
if assigned.sourceIndex != sourceIndex {
continue
}
for _, assigned := range rangesBySource[sourceIndex] {
pointer := assigned.path.String()
mode := sequenceHistoryAppend
if !seenSequences[pointer] {
@@ -149,6 +151,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node)
path: assigned.path.clone(), mode: mode, offset: assigned.start, length: assigned.end - assigned.start,
})
}
application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
applications = append(applications, application)
}
origins := projectedCompositionOrigins(inputs, ranges)
@@ -183,6 +186,7 @@ func completeCompositionOrigins(node *document.Node, path Path, origins map[stri
func (c *compositionClaims) append(node *document.Node, path Path, sourceIndex int) {
claim := compositionClaim{
node: node,
path: path.clone(),
kind: node.Kind,
location: publicLocation(node.Location),
@@ -213,7 +217,7 @@ type validatedSequence struct {
items []*document.Node
}
func validateRegisteredSequences(inputs []*document.Node, claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) {
itemsByPath := make(map[string]map[int]validatedSequence)
identitiesByPath := make(map[string]map[string]Location)
lengthsByPath := make(map[string]int)
@@ -226,10 +230,7 @@ func validateRegisteredSequences(inputs []*document.Node, claims []compositionCl
if !registered {
continue
}
node, ok := documentAtPath(inputs[claim.sourceIndex], claim.path)
if !ok || node.Kind != document.Sequence {
continue
}
node := claim.node
pathKey := claim.path.String()
identities := identitiesByPath[pathKey]
if identities == nil {
@@ -389,6 +390,19 @@ type sequenceRangeKey struct {
path string
}
type sequenceRangeIndex map[sequenceRangeKey]sequenceRange
func indexSequenceRanges(ranges []sequenceRange) sequenceRangeIndex {
if len(ranges) == 0 {
return nil
}
index := make(sequenceRangeIndex, len(ranges))
for _, assigned := range ranges {
index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned
}
return index
}
// Keep local paths for matching Stage 2 ranges and effective paths for emitted
// evidence. The paths are transformed segment by segment; pointer strings are
// only used as map keys after projection is complete.

47
internal/engine/compose_test.go
View File

@@ -1,6 +1,7 @@
package engine
import (
"fmt"
"reflect"
"strings"
"testing"
@@ -8,6 +9,52 @@ import (
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) {
policy := sequencePolicy{identityFields: make(map[string]string)}
var inputs []*document.Node
for source := 0; source < 3; source++ {
var text strings.Builder
for collection := 0; collection < 128; collection++ {
key := fmt.Sprintf("records%d", collection)
policy.identityFields[Keys(key).String()] = "id"
fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source)
}
inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String()))
}
composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
working := &document.Node{Kind: document.Mapping}
for _, application := range composed.applications {
applyHistoryApplication(&working, application, new(mappingIndex))
}
if !configurationValuesEqual(working, composed.root) {
t.Fatal("wide registered collections replay differs from assembly")
}
for collection := 0; collection < 128; collection++ {
path := Keys(fmt.Sprintf("records%d", collection)).Index(2).Key("id")
origin := composed.origins[path.String()]
if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-2" {
t.Fatalf("shifted origin at %s = %#v", path.String(), origin)
}
local, ok := historyApplicationLocalPath(composed.applications[2], path)
if !ok || local.String() != Keys(fmt.Sprintf("records%d", collection)).Index(0).Key("id").String() {
t.Fatal("wide local path projection failed")
}
}
clone := cloneComposedDocument(composed)
for _, original := range composed.applications[0].sequences {
pointer := original.path.String()
copy := clone.applications[0].sequencesByPath[pointer]
copy.path.segments[0].key = "changed"
if composed.applications[0].sequencesByPath[pointer].path.String() != pointer {
t.Fatal("history path index aliases cloned state")
}
break
}
}
func TestComposeDocumentsJoinsDisjointMappingsAndTracksClaims(t *testing.T) {
base := parseCompositionDocument(t, "source-1", `services:
api:

10
internal/engine/directives.go
View File

@@ -167,6 +167,10 @@ func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node
}
func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourceIndex int, ranges []sequenceRange) *document.Node {
return fragmentVersionClaimWithIndex(root, rule, sourceIndex, indexSequenceRanges(ranges))
}
func fragmentVersionClaimWithIndex(root *document.Node, rule *VersionRule, sourceIndex int, ranges sequenceRangeIndex) *document.Node {
if rule == nil {
return nil
}
@@ -192,15 +196,11 @@ func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourc
if !valid {
return current
}
for _, assigned := range ranges {
if assigned.sourceIndex != sourceIndex || assigned.path.String() != currentPath.String() {
continue
}
if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found {
if index < assigned.start || index >= assigned.end {
return nil
}
index -= assigned.start
break
}
if index < 0 || index >= len(current.Sequence) {
return nil

39
internal/engine/history.go
View File

@@ -45,11 +45,12 @@ func (c *retainedNodeCounter) add(root *document.Node) bool {
// Replay reconciles with canonical decoding before domain resolution so both
// contracts agree on the effective result without retaining a tree per field.
type historyApplication struct {
layer Layer
path Path
value *document.Node
origin Origin
sequences []sequenceHistoryContribution
layer Layer
path Path
value *document.Node
origin Origin
sequences []sequenceHistoryContribution
sequencesByPath map[string]sequenceHistoryContribution
}
func cloneHistoryApplications(values []historyApplication) []historyApplication {
@@ -62,6 +63,7 @@ func cloneHistoryApplications(values []historyApplication) []historyApplication
layer: value.layer, path: value.path.clone(), value: value.value,
origin: cloneOrigin(value.origin), sequences: cloneSequenceHistoryContributions(value.sequences),
}
cloned[index].sequencesByPath = indexSequenceHistoryContributions(cloned[index].sequences)
}
return cloned
}
@@ -86,7 +88,7 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef
walk(entry.Value, local.Key(entry.Key), effective.Key(entry.Key))
}
case document.Sequence:
contribution, projected := sequenceHistoryAt(application.sequences, local)
contribution, projected := sequenceHistoryAt(application.sequencesByPath, local)
for index, child := range node.Sequence {
effectiveIndex := index
if projected {
@@ -108,7 +110,7 @@ func historyApplicationLocalPath(application historyApplication, effective Path)
}
local := Path{}
for _, segment := range relative.segments {
if contribution, found := sequenceHistoryAt(application.sequences, local); found {
if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found {
index, valid := schemaIndex(segment)
if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length {
return Path{}, false
@@ -201,7 +203,7 @@ func pathsIntersect(left, right Path) bool {
func applyHistoryApplication(root **document.Node, application historyApplication, index *mappingIndex) {
if application.path.String() == "" {
mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequences)
mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath)
return
}
mergeHistoryAtPath(root, application.path, application.value, index)
@@ -211,7 +213,7 @@ func mergeHistoryDocument(target **document.Node, incoming *document.Node, index
mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil)
}
func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences []sequenceHistoryContribution) {
func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) {
if incoming == nil {
return
}
@@ -236,13 +238,20 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node,
}
}
func sequenceHistoryAt(contributions []sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
for _, contribution := range contributions {
if contribution.path.String() == path.String() {
return contribution, true
}
func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
contribution, found := contributions[path.String()]
return contribution, found
}
func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution {
if len(values) == 0 {
return nil
}
return sequenceHistoryContribution{}, false
index := make(map[string]sequenceHistoryContribution, len(values))
for _, value := range values {
index[value.path.String()] = value
}
return index
}
func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) {

33
internal/engine/imports.go
View File

@@ -564,11 +564,12 @@ func importedVersionClaim(documents []*document.Node, composed *composedDocument
if composed == nil || rule == nil {
return nil
}
ranges := indexSequenceRanges(composed.sequenceRanges)
for index, contribution := range documents {
if index == mainIndex {
continue
}
if node := fragmentVersionClaimProjected(contribution, rule, index, composed.sequenceRanges); node != nil {
if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil {
location := publicLocation(node.Location)
return &Diagnostic{
Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
@@ -595,18 +596,13 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
}
applications := composed.applications
projectedApplications := make([]historyApplication, 0, len(applications))
ranges := indexSequenceRanges(composed.sequenceRanges)
for sourceIndex, contribution := range scope.contributions {
if contribution.nested == nil {
projectedApplications = append(projectedApplications, applications[sourceIndex])
continue
}
start := 0
for _, assigned := range composed.sequenceRanges {
if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
start = assigned.start
break
}
}
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
prefix := contribution.destination.Index(start)
if sourceIndex < len(applications) {
outer := applications[sourceIndex]
@@ -622,6 +618,7 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
projected := nested
projected.path = appendDestination(prefix, nested.path)
projected.sequences = cloneSequenceHistoryContributions(nested.sequences)
projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences)
projectedApplications = append(projectedApplications, projected)
}
}
@@ -656,19 +653,14 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
}
var projectScope func(*composedDocument, acquisitionScope, Path, bool, string)
projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) {
ranges := indexSequenceRanges(currentComposition.sequenceRanges)
for sourceIndex, contribution := range current.contributions {
if contribution.source.ID == parentSourceID {
continue
}
contributionPrefix := prefix
contributionInsideItem := insideItem
start := 0
for _, assigned := range currentComposition.sequenceRanges {
if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
start = assigned.start
break
}
}
start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start
if contribution.source.placementMode == placementModeSequenceItem {
container := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = container.clone()
@@ -692,7 +684,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
}
projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex)
} else {
projectSourceVersionEvidence(contribution.processing, contribution.destination, currentComposition.sequenceRanges, sourceIndex)
projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex)
}
if contribution.nested != nil && contribution.nested.acquisition != nil {
for _, nestedScope := range contribution.nested.acquisition {
@@ -704,7 +696,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc
projectScope(composed, scope, Path{}, false, "")
}
func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges []sequenceRange, sourceIndex int) {
func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) {
if processing != nil && processing.evidence != nil {
evidence := processing.evidence
evidence.destination = destination.clone()
@@ -713,11 +705,12 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des
evidence.projectedPaths = nil
paths := evidence.sensitivityPaths()
for index, path := range paths {
for _, assigned := range ranges {
if assigned.sourceIndex != sourceIndex || !pathContains(assigned.path, path) || len(path.segments) <= len(assigned.path.segments) {
for position := range path.segments {
prefix := Path{segments: path.segments[:position]}
assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}]
if !found {
continue
}
position := len(assigned.path.segments)
item, ok := schemaIndex(path.segments[position])
if ok && path.segments[position].kind != pathSegmentKey {
path.segments[position] = pathSegment{kind: pathSegmentIndex, index: assigned.start + item}

3
internal/engine/load_request.go
View File

@@ -196,8 +196,9 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
return nil, sources, contexts, diagnostics
}
if composition != nil && rule != nil {
ranges := indexSequenceRanges(composition.sequenceRanges)
for sourceIndex, document := range documents {
if node := fragmentVersionClaimProjected(document, rule, sourceIndex, composition.sequenceRanges); node != nil {
if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
location := publicLocation(node.Location)
return nil, sources, contexts, []Diagnostic{{
Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),