diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 5133839..64e2ac8 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -222,7 +222,15 @@ mapping is still explicit. Defaults-only overrides keep their existing behavior. ## 7. Remove new quadratic scans from sequence construction — P2 -- [ ] Addressed and behaviorally validated. +- [x] Addressed and behaviorally validated. + +Completion: claims retain encountered immutable nodes; history construction +groups ranges by source; scope/version projection uses source/path range +lookups; retained-history traversal and replay use per-application path indexes. +Ordered slices and detached clone ownership remain intact. Static review confirms +the listed nested scans are removed. A wide-collection replay/origin/ownership +regression, the everyday check, and `go test -race ./...` passed. No timing +threshold or release gate was added. **Owners:** [compose.go](../../internal/engine/compose.go), especially `validateRegisteredSequences` and application construction; range lookups in diff --git a/internal/engine/compose.go b/internal/engine/compose.go index 31f6c83..a0c2401 100644 --- a/internal/engine/compose.go +++ b/internal/engine/compose.go @@ -59,6 +59,7 @@ func cloneComposedDocument(base *composedDocument) *composedDocument { } type compositionClaim struct { + node *document.Node // borrowed immutable contribution, not the assembly tree path Path kind document.Kind location Location @@ -121,7 +122,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) return nil, []Diagnostic{compositionConflictDiagnostic(claim.path, claims)} } - validated, ranges, diagnostics := validateRegisteredSequences(inputs, collected.ordered, policy) + validated, ranges, diagnostics := validateRegisteredSequences(collected.ordered, policy) if len(diagnostics) != 0 { return nil, diagnostics } @@ -133,12 +134,13 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) } applications := make([]historyApplication, 0, len(inputs)) seenSequences := make(map[string]bool) + rangesBySource := make([][]sequenceRange, len(inputs)) + for _, assigned := range ranges { + rangesBySource[assigned.sourceIndex] = append(rangesBySource[assigned.sourceIndex], assigned) + } for sourceIndex, input := range inputs { application := historyApplication{layer: LayerBase, value: input} - for _, assigned := range ranges { - if assigned.sourceIndex != sourceIndex { - continue - } + for _, assigned := range rangesBySource[sourceIndex] { pointer := assigned.path.String() mode := sequenceHistoryAppend if !seenSequences[pointer] { @@ -149,6 +151,7 @@ func composeDocumentsWithPolicy(policy sequencePolicy, inputs ...*document.Node) path: assigned.path.clone(), mode: mode, offset: assigned.start, length: assigned.end - assigned.start, }) } + application.sequencesByPath = indexSequenceHistoryContributions(application.sequences) applications = append(applications, application) } origins := projectedCompositionOrigins(inputs, ranges) @@ -183,6 +186,7 @@ func completeCompositionOrigins(node *document.Node, path Path, origins map[stri func (c *compositionClaims) append(node *document.Node, path Path, sourceIndex int) { claim := compositionClaim{ + node: node, path: path.clone(), kind: node.Kind, location: publicLocation(node.Location), @@ -213,7 +217,7 @@ type validatedSequence struct { items []*document.Node } -func validateRegisteredSequences(inputs []*document.Node, claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) { +func validateRegisteredSequences(claims []compositionClaim, policy sequencePolicy) (map[string]map[int]validatedSequence, []sequenceRange, []Diagnostic) { itemsByPath := make(map[string]map[int]validatedSequence) identitiesByPath := make(map[string]map[string]Location) lengthsByPath := make(map[string]int) @@ -226,10 +230,7 @@ func validateRegisteredSequences(inputs []*document.Node, claims []compositionCl if !registered { continue } - node, ok := documentAtPath(inputs[claim.sourceIndex], claim.path) - if !ok || node.Kind != document.Sequence { - continue - } + node := claim.node pathKey := claim.path.String() identities := identitiesByPath[pathKey] if identities == nil { @@ -389,6 +390,19 @@ type sequenceRangeKey struct { path string } +type sequenceRangeIndex map[sequenceRangeKey]sequenceRange + +func indexSequenceRanges(ranges []sequenceRange) sequenceRangeIndex { + if len(ranges) == 0 { + return nil + } + index := make(sequenceRangeIndex, len(ranges)) + for _, assigned := range ranges { + index[sequenceRangeKey{sourceIndex: assigned.sourceIndex, path: assigned.path.String()}] = assigned + } + return index +} + // Keep local paths for matching Stage 2 ranges and effective paths for emitted // evidence. The paths are transformed segment by segment; pointer strings are // only used as map keys after projection is complete. diff --git a/internal/engine/compose_test.go b/internal/engine/compose_test.go index b6c5ac4..ec9aba7 100644 --- a/internal/engine/compose_test.go +++ b/internal/engine/compose_test.go @@ -1,6 +1,7 @@ package engine import ( + "fmt" "reflect" "strings" "testing" @@ -8,6 +9,52 @@ import ( "gitea.maximumdirect.net/eric/ConfigKit/internal/document" ) +func TestComposeWideRegisteredCollectionsPreservesEvidenceAndReplay(t *testing.T) { + policy := sequencePolicy{identityFields: make(map[string]string)} + var inputs []*document.Node + for source := 0; source < 3; source++ { + var text strings.Builder + for collection := 0; collection < 128; collection++ { + key := fmt.Sprintf("records%d", collection) + policy.identityFields[Keys(key).String()] = "id" + fmt.Fprintf(&text, "%s: [{id: source%d}]\n", key, source) + } + inputs = append(inputs, parseCompositionDocument(t, fmt.Sprintf("source-%d", source), text.String())) + } + composed, diagnostics := composeDocumentsWithPolicy(policy, inputs...) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + working := &document.Node{Kind: document.Mapping} + for _, application := range composed.applications { + applyHistoryApplication(&working, application, new(mappingIndex)) + } + if !configurationValuesEqual(working, composed.root) { + t.Fatal("wide registered collections replay differs from assembly") + } + for collection := 0; collection < 128; collection++ { + path := Keys(fmt.Sprintf("records%d", collection)).Index(2).Key("id") + origin := composed.origins[path.String()] + if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-2" { + t.Fatalf("shifted origin at %s = %#v", path.String(), origin) + } + local, ok := historyApplicationLocalPath(composed.applications[2], path) + if !ok || local.String() != Keys(fmt.Sprintf("records%d", collection)).Index(0).Key("id").String() { + t.Fatal("wide local path projection failed") + } + } + clone := cloneComposedDocument(composed) + for _, original := range composed.applications[0].sequences { + pointer := original.path.String() + copy := clone.applications[0].sequencesByPath[pointer] + copy.path.segments[0].key = "changed" + if composed.applications[0].sequencesByPath[pointer].path.String() != pointer { + t.Fatal("history path index aliases cloned state") + } + break + } +} + func TestComposeDocumentsJoinsDisjointMappingsAndTracksClaims(t *testing.T) { base := parseCompositionDocument(t, "source-1", `services: api: diff --git a/internal/engine/directives.go b/internal/engine/directives.go index 73ababb..4f1a0e7 100644 --- a/internal/engine/directives.go +++ b/internal/engine/directives.go @@ -167,6 +167,10 @@ func fragmentVersionClaim(root *document.Node, rule *VersionRule) *document.Node } func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourceIndex int, ranges []sequenceRange) *document.Node { + return fragmentVersionClaimWithIndex(root, rule, sourceIndex, indexSequenceRanges(ranges)) +} + +func fragmentVersionClaimWithIndex(root *document.Node, rule *VersionRule, sourceIndex int, ranges sequenceRangeIndex) *document.Node { if rule == nil { return nil } @@ -192,15 +196,11 @@ func fragmentVersionClaimProjected(root *document.Node, rule *VersionRule, sourc if !valid { return current } - for _, assigned := range ranges { - if assigned.sourceIndex != sourceIndex || assigned.path.String() != currentPath.String() { - continue - } + if assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: currentPath.String()}]; found { if index < assigned.start || index >= assigned.end { return nil } index -= assigned.start - break } if index < 0 || index >= len(current.Sequence) { return nil diff --git a/internal/engine/history.go b/internal/engine/history.go index 54db081..d76a8f6 100644 --- a/internal/engine/history.go +++ b/internal/engine/history.go @@ -45,11 +45,12 @@ func (c *retainedNodeCounter) add(root *document.Node) bool { // Replay reconciles with canonical decoding before domain resolution so both // contracts agree on the effective result without retaining a tree per field. type historyApplication struct { - layer Layer - path Path - value *document.Node - origin Origin - sequences []sequenceHistoryContribution + layer Layer + path Path + value *document.Node + origin Origin + sequences []sequenceHistoryContribution + sequencesByPath map[string]sequenceHistoryContribution } func cloneHistoryApplications(values []historyApplication) []historyApplication { @@ -62,6 +63,7 @@ func cloneHistoryApplications(values []historyApplication) []historyApplication layer: value.layer, path: value.path.clone(), value: value.value, origin: cloneOrigin(value.origin), sequences: cloneSequenceHistoryContributions(value.sequences), } + cloned[index].sequencesByPath = indexSequenceHistoryContributions(cloned[index].sequences) } return cloned } @@ -86,7 +88,7 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef walk(entry.Value, local.Key(entry.Key), effective.Key(entry.Key)) } case document.Sequence: - contribution, projected := sequenceHistoryAt(application.sequences, local) + contribution, projected := sequenceHistoryAt(application.sequencesByPath, local) for index, child := range node.Sequence { effectiveIndex := index if projected { @@ -108,7 +110,7 @@ func historyApplicationLocalPath(application historyApplication, effective Path) } local := Path{} for _, segment := range relative.segments { - if contribution, found := sequenceHistoryAt(application.sequences, local); found { + if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found { index, valid := schemaIndex(segment) if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length { return Path{}, false @@ -201,7 +203,7 @@ func pathsIntersect(left, right Path) bool { func applyHistoryApplication(root **document.Node, application historyApplication, index *mappingIndex) { if application.path.String() == "" { - mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequences) + mergeHistoryDocumentAtPath(root, application.value, Path{}, index, application.sequencesByPath) return } mergeHistoryAtPath(root, application.path, application.value, index) @@ -211,7 +213,7 @@ func mergeHistoryDocument(target **document.Node, incoming *document.Node, index mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil) } -func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences []sequenceHistoryContribution) { +func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) { if incoming == nil { return } @@ -236,13 +238,20 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, } } -func sequenceHistoryAt(contributions []sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) { - for _, contribution := range contributions { - if contribution.path.String() == path.String() { - return contribution, true - } +func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) { + contribution, found := contributions[path.String()] + return contribution, found +} + +func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution { + if len(values) == 0 { + return nil } - return sequenceHistoryContribution{}, false + index := make(map[string]sequenceHistoryContribution, len(values)) + for _, value := range values { + index[value.path.String()] = value + } + return index } func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) { diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 1ebb04a..43fd597 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -564,11 +564,12 @@ func importedVersionClaim(documents []*document.Node, composed *composedDocument if composed == nil || rule == nil { return nil } + ranges := indexSequenceRanges(composed.sequenceRanges) for index, contribution := range documents { if index == mainIndex { continue } - if node := fragmentVersionClaimProjected(contribution, rule, index, composed.sequenceRanges); node != nil { + if node := fragmentVersionClaimWithIndex(contribution, rule, index, ranges); node != nil { location := publicLocation(node.Location) return &Diagnostic{ Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(), @@ -595,18 +596,13 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { } applications := composed.applications projectedApplications := make([]historyApplication, 0, len(applications)) + ranges := indexSequenceRanges(composed.sequenceRanges) for sourceIndex, contribution := range scope.contributions { if contribution.nested == nil { projectedApplications = append(projectedApplications, applications[sourceIndex]) continue } - start := 0 - for _, assigned := range composed.sequenceRanges { - if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() { - start = assigned.start - break - } - } + start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start prefix := contribution.destination.Index(start) if sourceIndex < len(applications) { outer := applications[sourceIndex] @@ -622,6 +618,7 @@ func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { projected := nested projected.path = appendDestination(prefix, nested.path) projected.sequences = cloneSequenceHistoryContributions(nested.sequences) + projected.sequencesByPath = indexSequenceHistoryContributions(projected.sequences) projectedApplications = append(projectedApplications, projected) } } @@ -656,19 +653,14 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc } var projectScope func(*composedDocument, acquisitionScope, Path, bool, string) projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) { + ranges := indexSequenceRanges(currentComposition.sequenceRanges) for sourceIndex, contribution := range current.contributions { if contribution.source.ID == parentSourceID { continue } contributionPrefix := prefix contributionInsideItem := insideItem - start := 0 - for _, assigned := range currentComposition.sequenceRanges { - if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() { - start = assigned.start - break - } - } + start := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: contribution.destination.String()}].start if contribution.source.placementMode == placementModeSequenceItem { container := appendDestination(contributionPrefix, contribution.destination) contribution.source.Destination = container.clone() @@ -692,7 +684,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc } projectSourceVersionEvidence(contribution.processing, exactDestination, nil, sourceIndex) } else { - projectSourceVersionEvidence(contribution.processing, contribution.destination, currentComposition.sequenceRanges, sourceIndex) + projectSourceVersionEvidence(contribution.processing, contribution.destination, ranges, sourceIndex) } if contribution.nested != nil && contribution.nested.acquisition != nil { for _, nestedScope := range contribution.nested.acquisition { @@ -704,7 +696,7 @@ func projectAcquiredDestinations(composed *composedDocument, scope acquisitionSc projectScope(composed, scope, Path{}, false, "") } -func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges []sequenceRange, sourceIndex int) { +func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, destination Path, ranges sequenceRangeIndex, sourceIndex int) { if processing != nil && processing.evidence != nil { evidence := processing.evidence evidence.destination = destination.clone() @@ -713,11 +705,12 @@ func projectSourceVersionEvidence(processing *sourceVersionProcessingResult, des evidence.projectedPaths = nil paths := evidence.sensitivityPaths() for index, path := range paths { - for _, assigned := range ranges { - if assigned.sourceIndex != sourceIndex || !pathContains(assigned.path, path) || len(path.segments) <= len(assigned.path.segments) { + for position := range path.segments { + prefix := Path{segments: path.segments[:position]} + assigned, found := ranges[sequenceRangeKey{sourceIndex: sourceIndex, path: prefix.String()}] + if !found { continue } - position := len(assigned.path.segments) item, ok := schemaIndex(path.segments[position]) if ok && path.segments[position].kind != pathSegmentKey { path.segments[position] = pathSegment{kind: pathSegmentIndex, index: assigned.start + item} diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index 5ceaf4a..469ea82 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -196,8 +196,9 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver return nil, sources, contexts, diagnostics } if composition != nil && rule != nil { + ranges := indexSequenceRanges(composition.sequenceRanges) for sourceIndex, document := range documents { - if node := fragmentVersionClaimProjected(document, rule, sourceIndex, composition.sequenceRanges); node != nil { + if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil { location := publicLocation(node.Location) return nil, sources, contexts, []Diagnostic{{ Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),