Extend history replay for mapped sequence items

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Sequence Identity And Combination Implementation Plan
 
-Status: implementation in progress; Stages 1–6 complete.
+Status: implementation in progress; Stages 1–7 complete.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. This plan starts at commit
@@ -337,6 +337,9 @@ projection consumers or public precedence.
 Validation: sequence/history/replay tests and the everyday check. Keep current
 append/replace callers working through the extended descriptors.
 
+**Completion:** Implemented and verified with focused sequence-history tests and
+`./scripts/check.sh`.
+
 ## Stage 8 — Project Sequence Origins And Inspection Evidence
 
 Scope: use the prepared item mappings consistently in metadata consumers.

internal/engine/compose.go

@@ -23,13 +23,16 @@ type sequenceHistoryMode uint8
 const (
 	sequenceHistoryReplace sequenceHistoryMode = iota + 1
 	sequenceHistoryAppend
+	sequenceHistoryMapped
 )
 
 type sequenceHistoryContribution struct {
-	path   Path
+	path         Path
-	mode   sequenceHistoryMode
+	mode         sequenceHistoryMode
-	offset int
+	offset       int
-	length int
+	length       int
+	items        []sequenceItemProjection
+	replacements []sequenceItemReplacement
 }
 
 // assemblyAccumulator owns one mutable working document. Acquired trees held by

internal/engine/history.go

@@ -50,7 +50,7 @@ type historyApplication struct {
 	value           *document.Node
 	origin          Origin
 	sequences       []sequenceHistoryContribution
-	sequencesByPath map[string]sequenceHistoryContribution
+	sequencesByPath map[string]sequenceHistoryContributionIndex
 }
 
 func cloneHistoryApplications(values []historyApplication) []historyApplication {
@@ -92,7 +92,14 @@ func walkHistoryApplication(application historyApplication, visit func(local, ef
 			for index, child := range node.Sequence {
 				effectiveIndex := index
 				if projected {
-					effectiveIndex += contribution.offset
+					if contribution.mode == sequenceHistoryMapped {
+						effectiveIndex = contribution.outputForInput(index)
+						if effectiveIndex < 0 {
+							continue
+						}
+					} else {
+						effectiveIndex += contribution.offset
+					}
 				}
 				walk(child, local.Index(index), effective.Index(effectiveIndex))
 			}
@@ -112,7 +119,18 @@ func historyApplicationLocalPath(application historyApplication, effective Path)
 	for _, segment := range relative.segments {
 		if contribution, found := sequenceHistoryAt(application.sequencesByPath, local); found {
 			index, valid := schemaIndex(segment)
-			if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length {
+			if !valid || segment.kind == pathSegmentKey {
+				return Path{}, false
+			}
+			if contribution.mode == sequenceHistoryMapped {
+				inputIndex := contribution.inputForOutput(index)
+				if inputIndex < 0 {
+					return Path{}, false
+				}
+				local = local.Index(inputIndex)
+				continue
+			}
+			if index < contribution.offset || index >= contribution.offset+contribution.length {
 				return Path{}, false
 			}
 			local = local.Index(index - contribution.offset)
@@ -149,6 +167,16 @@ func applicationOperationPath(application historyApplication, query Path, lookup
 					return container, true
 				}
 			}
+			indexed, _ := sequenceHistoryAt(application.sequencesByPath, contribution.path)
+			if indexed.mode == sequenceHistoryMapped && len(query.segments) > len(container.segments) {
+				outputIndex, valid := schemaIndex(query.segments[len(container.segments)])
+				_, replaced := indexed.replacedOutputs[outputIndex]
+				if valid && replaced {
+					if _, present := documentAtPathWithLookup(application.value, local, lookup); !ok || !present {
+						return container.Index(outputIndex), true
+					}
+				}
+			}
 			if !ok {
 				continue
 			}
@@ -213,7 +241,7 @@ func mergeHistoryDocument(target **document.Node, incoming *document.Node, index
 	mergeHistoryDocumentAtPath(target, incoming, Path{}, index, nil)
 }
 
-func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContribution) {
+func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node, path Path, index *mappingIndex, sequences map[string]sequenceHistoryContributionIndex) {
 	if incoming == nil {
 		return
 	}
@@ -238,23 +266,71 @@ func mergeHistoryDocumentAtPath(target **document.Node, incoming *document.Node,
 	}
 }
 
-func sequenceHistoryAt(contributions map[string]sequenceHistoryContribution, path Path) (sequenceHistoryContribution, bool) {
+func sequenceHistoryAt(contributions map[string]sequenceHistoryContributionIndex, path Path) (sequenceHistoryContributionIndex, bool) {
 	contribution, found := contributions[path.String()]
 	return contribution, found
 }
 
-func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContribution {
+type sequenceHistoryContributionIndex struct {
+	sequenceHistoryContribution
+	outputIndexes   []int
+	inputIndexes    []int
+	replacedOutputs map[int]struct{}
+}
+
+func indexSequenceHistoryContributions(values []sequenceHistoryContribution) map[string]sequenceHistoryContributionIndex {
 	if len(values) == 0 {
 		return nil
 	}
-	index := make(map[string]sequenceHistoryContribution, len(values))
+	index := make(map[string]sequenceHistoryContributionIndex, len(values))
 	for _, value := range values {
-		index[value.path.String()] = value
+		contribution := sequenceHistoryContributionIndex{sequenceHistoryContribution: value}
+		if value.mode == sequenceHistoryMapped {
+			maxOutput := -1
+			for _, projection := range value.items {
+				if projection.outputIndex > maxOutput {
+					maxOutput = projection.outputIndex
+				}
+			}
+			contribution.outputIndexes = make([]int, len(value.items))
+			for inputIndex := range contribution.outputIndexes {
+				contribution.outputIndexes[inputIndex] = -1
+			}
+			contribution.inputIndexes = make([]int, maxOutput+1)
+			for outputIndex := range contribution.inputIndexes {
+				contribution.inputIndexes[outputIndex] = -1
+			}
+			for _, projection := range value.items {
+				if projection.inputIndex >= 0 && projection.inputIndex < len(contribution.outputIndexes) && projection.outputIndex >= 0 && projection.outputIndex < len(contribution.inputIndexes) {
+					contribution.outputIndexes[projection.inputIndex] = projection.outputIndex
+					contribution.inputIndexes[projection.outputIndex] = projection.inputIndex
+				}
+			}
+			contribution.replacedOutputs = make(map[int]struct{}, len(value.replacements))
+			for _, replacement := range value.replacements {
+				contribution.replacedOutputs[replacement.outputIndex] = struct{}{}
+			}
+		}
+		index[value.path.String()] = contribution
 	}
 	return index
 }
 
-func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContribution) {
+func (contribution sequenceHistoryContributionIndex) outputForInput(inputIndex int) int {
+	if inputIndex < 0 || inputIndex >= len(contribution.outputIndexes) {
+		return -1
+	}
+	return contribution.outputIndexes[inputIndex]
+}
+
+func (contribution sequenceHistoryContributionIndex) inputForOutput(outputIndex int) int {
+	if outputIndex < 0 || outputIndex >= len(contribution.inputIndexes) {
+		return -1
+	}
+	return contribution.inputIndexes[outputIndex]
+}
+
+func applySequenceHistory(target **document.Node, incoming *document.Node, contribution sequenceHistoryContributionIndex) {
 	if incoming == nil || incoming.Kind != document.Sequence {
 		return
 	}
@@ -262,6 +338,18 @@ func applySequenceHistory(target **document.Node, incoming *document.Node, contr
 		*target = document.Clone(incoming)
 		return
 	}
+	if contribution.mode == sequenceHistoryMapped {
+		for _, projection := range contribution.items {
+			if projection.inputIndex < 0 || projection.inputIndex >= len(incoming.Sequence) || projection.outputIndex < 0 {
+				continue
+			}
+			for len((*target).Sequence) <= projection.outputIndex {
+				(*target).Sequence = append((*target).Sequence, nil)
+			}
+			(*target).Sequence[projection.outputIndex] = document.Clone(incoming.Sequence[projection.inputIndex])
+		}
+		return
+	}
 	for _, item := range incoming.Sequence {
 		(*target).Sequence = append((*target).Sequence, document.Clone(item))
 	}
@@ -275,6 +363,8 @@ func cloneSequenceHistoryContributions(values []sequenceHistoryContribution) []s
 	for index, value := range values {
 		cloned[index] = value
 		cloned[index].path = value.path.clone()
+		cloned[index].items = append([]sequenceItemProjection(nil), value.items...)
+		cloned[index].replacements = append([]sequenceItemReplacement(nil), value.replacements...)
 	}
 	return cloned
 }

internal/engine/history_sequence_test.go

@@ -230,6 +230,170 @@ func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
 	}
 }
 
+func TestSequenceHistoryReplaysCombinationEvidence(t *testing.T) {
+	tests := []struct {
+		name      string
+		operation sequenceMode
+		lower     string
+		higher    string
+	}{
+		{
+			name:      "append",
+			operation: sequenceModeAppend,
+			lower:     "records: [{id: first}, {id: last}]\n",
+			higher:    "records: [{id: added}]\n",
+		},
+		{
+			name:      "upsert",
+			operation: sequenceModeUpsert,
+			lower: `records:
+  - id: first
+    label: same
+    nested: {removed: lower}
+  - id: middle
+  - id: last
+    nested: {removed: lower}
+`,
+			higher: `records:
+  - id: last
+    nested: {kept: higher}
+  - id: first
+    label: same
+  - id: added
+`,
+		},
+		{
+			name:      "replace",
+			operation: sequenceModeReplace,
+			lower:     "records: [{id: discarded}]\n",
+			higher:    "records: [{id: replacement}]\n",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			lowerRoot := parseCompositionDocument(t, "lower", test.lower)
+			higherRoot := parseCompositionDocument(t, "higher", test.higher)
+			lower, _ := documentAtPath(lowerRoot, Keys("records"))
+			higher, _ := documentAtPath(higherRoot, Keys("records"))
+			fields := []string(nil)
+			if test.operation != sequenceModeReplace {
+				fields = []string{"id"}
+			}
+			combined, diagnostics := combineSequence(lower, higher, sequenceCombinationPolicy(test.operation, fields...), sequenceCombinationContext{destination: Keys("records")})
+			if len(diagnostics) != 0 {
+				t.Fatalf("combine diagnostics = %#v", diagnostics)
+			}
+
+			mode := sequenceHistoryMapped
+			if test.operation == sequenceModeReplace {
+				mode = sequenceHistoryReplace
+			}
+			length := 0
+			if mode == sequenceHistoryReplace {
+				length = len(combined.higherItems)
+			}
+			application := historyApplication{
+				layer: LayerOverlay,
+				value: higherRoot,
+				sequences: []sequenceHistoryContribution{{
+					path: Keys("records"), mode: mode, length: length,
+					items: combined.higherItems, replacements: combined.replacements,
+				}},
+			}
+			application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
+			replayed := document.Clone(lowerRoot)
+			applyHistoryApplication(&replayed, application, new(mappingIndex))
+			got, present := documentAtPath(replayed, Keys("records"))
+			if !present || !reflect.DeepEqual(got.Sequence, combined.value.Sequence) {
+				t.Fatalf("replayed sequence = %#v, want combination = %#v", got, combined.value.Sequence)
+			}
+		})
+	}
+}
+
+func TestMappedSequenceHistoryReplacesWholeRecordsAndKeepsDisplacedEvidence(t *testing.T) {
+	lowerRoot := parseCompositionDocument(t, "lower", `records:
+  - id: first
+    label: same
+  - id: middle
+  - id: last
+    nested: {removed: LOWER-CANARY}
+`)
+	higherRoot := parseCompositionDocument(t, "higher", `records:
+  - id: last
+    nested: {kept: higher}
+  - id: first
+    label: same
+  - id: added
+`)
+	lower, _ := documentAtPath(lowerRoot, Keys("records"))
+	higher, _ := documentAtPath(higherRoot, Keys("records"))
+	combined, diagnostics := combineSequence(
+		lower, higher,
+		sequenceCombinationPolicy(sequenceModeUpsert, "id"),
+		sequenceCombinationContext{destination: Keys("records")},
+	)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	application := historyApplication{
+		layer: LayerOverlay, value: higherRoot,
+		sequences: []sequenceHistoryContribution{{
+			path: Keys("records"), mode: sequenceHistoryMapped,
+			items: combined.higherItems, replacements: combined.replacements,
+		}},
+	}
+	application.sequencesByPath = indexSequenceHistoryContributions(application.sequences)
+	clonedApplications := cloneHistoryApplications([]historyApplication{application})
+	clonedApplications[0].sequences[0].items[0].outputIndex = 99
+	clonedApplications[0].sequences[0].replacements[0].outputIndex = 99
+	if application.sequences[0].items[0].outputIndex == 99 || application.sequences[0].replacements[0].outputIndex == 99 {
+		t.Fatal("cloned mapped history aliases its source projections")
+	}
+
+	query := Keys("records").Index(2).Key("nested").Key("removed")
+	operationPath, relevant := applicationOperationPath(application, query, nil)
+	if !relevant || operationPath.String() != "/records/2" {
+		t.Fatalf("removed descendant operation = %s, %v", operationPath.String(), relevant)
+	}
+	localPath, ok := historyApplicationLocalPath(application, Keys("records").Index(2).Key("nested").Key("kept"))
+	if !ok || localPath.String() != "/records/0/nested/kept" {
+		t.Fatalf("mapped effective path localizes to %s, %v", localPath.String(), ok)
+	}
+	origin := historyApplicationOrigin(application, operationPath, nil)
+	if len(origin.Locations) != 1 || origin.Locations[0].SourceID != "higher" {
+		t.Fatalf("replacement operation origin = %#v", origin)
+	}
+	var walkedMappedPath bool
+	walkHistoryApplication(application, func(local, effective Path, _ *document.Node) {
+		if local.String() == "/records/0/nested/kept" && effective.String() == "/records/2/nested/kept" {
+			walkedMappedPath = true
+		}
+	})
+	if !walkedMappedPath {
+		t.Fatal("history walk did not project a noncontiguous incoming item")
+	}
+
+	replayed := document.Clone(lowerRoot)
+	applyHistoryApplication(&replayed, application, new(mappingIndex))
+	sequence, _ := documentAtPath(replayed, Keys("records"))
+	if _, present := documentAtPath(sequence.Sequence[2], Keys("nested", "removed")); present {
+		t.Fatal("whole-record replacement retained an omitted descendant")
+	}
+	if value, present := documentAtPath(sequence.Sequence[2], Keys("nested", "kept")); !present || value.Text != "higher" {
+		t.Fatalf("replacement descendant = %#v, %v", value, present)
+	}
+	if sequence.Sequence[1].Mapping[0].Value.Text != "middle" {
+		t.Fatal("unchanged neighboring record moved or changed")
+	}
+	if value, present := documentAtPath(lowerRoot, Keys("records").Index(2).Key("nested").Key("removed")); !present || value.Text != "LOWER-CANARY" {
+		t.Fatalf("displaced lower contribution was not retained as raw history: %#v, %v", value, present)
+	}
+	if got := application.sequences[0].replacements; !reflect.DeepEqual(got, combined.replacements) || len(got) != 2 {
+		t.Fatalf("replacement evidence lost equal-valued or noncontiguous events: %#v", got)
+	}
+}
+
 func mustParsePath(t *testing.T, value string) Path {
 	t.Helper()
 	path, err := ParsePath(value)