Project history and sensitivity paths

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Implementation Plan: Sequence Placement and Composition
 
-Status: implementation in progress; stages 1–4 completed.
+Status: implementation in progress; stages 1–5 completed.
 
 ## Purpose and authority
 
@@ -372,6 +372,22 @@ Validation: metadata/provenance/history/explanation/secret tests,
 `go test -race ./internal/engine`, and `./scripts/check.sh`.
 Completion: the private registered composer can feed existing inspection safely.
 
+Completion evidence: history applications now expose one source-local/effective
+path traversal used by metadata enumeration, explanation replay, origin lookup,
+and secret classification. Appended records and nested descendants use assigned
+indexes while retained nodes and source locations remain local. Secret
+classification evaluates projected historical shapes before folding sensitivity
+by pointer, preserving numeric key/index distinctions and redacting replaced
+records. Tests cover shifted metadata/history/rendering, origin attribution, and
+redaction after replacement. These checks passed with Go `go1.26.9 linux/amd64`:
+
+- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'Metadata|Provenance|History|Explain|Secret' -count=1`
+- `GOTOOLCHAIN=local GOPROXY=off go test -race ./internal/engine`
+- `GOTOOLCHAIN=local ./scripts/check.sh`
+- `git diff --check`
+
+No environmental limitation affected validation; checks ran on Linux/amd64.
+
 ## Stage 6 — Refactor acquisition into composition scopes without new behavior
 
 Scope: `imports.go`, `load_input.go`, `load_request.go`, and their handoffs.

internal/engine/history.go

@@ -66,6 +66,61 @@ func cloneHistoryApplications(values []historyApplication) []historyApplication
 	return cloned
 }
 
+// walkHistoryApplication visits a retained source tree with both its local
+// semantic path and its effective projected path. Sequence fragments keep
+// their original tree indexes, while the effective path uses the assigned
+// range recorded by strict composition.
+func walkHistoryApplication(application historyApplication, visit func(local, effective Path, node *document.Node)) {
+	if application.value == nil {
+		return
+	}
+	var walk func(*document.Node, Path, Path)
+	walk = func(node *document.Node, local, effective Path) {
+		if node == nil {
+			return
+		}
+		visit(local, appendDestination(application.path, effective), node)
+		switch node.Kind {
+		case document.Mapping:
+			for _, entry := range node.Mapping {
+				walk(entry.Value, local.Key(entry.Key), effective.Key(entry.Key))
+			}
+		case document.Sequence:
+			contribution, projected := sequenceHistoryAt(application.sequences, local)
+			for index, child := range node.Sequence {
+				effectiveIndex := index
+				if projected {
+					effectiveIndex += contribution.offset
+				}
+				walk(child, local.Index(index), effective.Index(effectiveIndex))
+			}
+		}
+	}
+	walk(application.value, Path{}, Path{})
+}
+
+// historyApplicationLocalPath translates an effective projected path back to
+// the retained source tree, including indexes shifted by enclosing appends.
+func historyApplicationLocalPath(application historyApplication, effective Path) (Path, bool) {
+	relative, ok := relativeHistoryPath(application.path, effective)
+	if !ok {
+		return Path{}, false
+	}
+	local := Path{}
+	for _, segment := range relative.segments {
+		if contribution, found := sequenceHistoryAt(application.sequences, local); found {
+			index, valid := schemaIndex(segment)
+			if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length {
+				return Path{}, false
+			}
+			local = local.Index(index - contribution.offset)
+			continue
+		}
+		local.segments = append(local.segments, segment)
+	}
+	return local, true
+}
+
 func applicationOperationPath(application historyApplication, query Path, lookup *documentLookup) (Path, bool) {
 	if !application.path.valid() || application.value == nil {
 		return Path{}, false
@@ -78,14 +133,13 @@ func applicationOperationPath(application historyApplication, query Path, lookup
 	}
 	sequencePathMatched := false
 	for _, contribution := range application.sequences {
-		if !pathContains(contribution.path, query) && !pathContains(query, contribution.path) {
+		container := appendDestination(application.path, contribution.path)
+		if !pathContains(container, query) && !pathContains(query, container) {
 			continue
 		}
 		sequencePathMatched = true
-		if pathContains(contribution.path, query) && len(query.segments) > len(contribution.path.segments) {
+		if pathContains(container, query) && len(query.segments) > len(container.segments) {
-			segment := query.segments[len(contribution.path.segments)]
+			if _, ok := historyApplicationLocalPath(application, query); !ok {
-			itemIndex, ok := schemaIndex(segment)
-			if !ok || itemIndex < contribution.offset || itemIndex >= contribution.offset+contribution.length {
 				continue
 			}
 		}
@@ -115,26 +169,10 @@ func historyApplicationOrigin(application historyApplication, operationPath Path
 		return cloneOrigin(application.origin)
 	}
 	origin := Origin{Layer: application.layer}
-	relative, ok := relativeHistoryPath(application.path, operationPath)
+	relative, ok := historyApplicationLocalPath(application, operationPath)
 	if !ok {
 		return origin
 	}
-	for _, contribution := range application.sequences {
-		if !pathContains(contribution.path, operationPath) {
-			continue
-		}
-		relative, ok = relativeHistoryPath(Path{}, operationPath)
-		if !ok || len(relative.segments) < len(contribution.path.segments)+1 {
-			break
-		}
-		itemPosition := len(contribution.path.segments)
-		itemIndex, valid := schemaIndex(relative.segments[itemPosition])
-		if !valid || itemIndex < contribution.offset || itemIndex >= contribution.offset+contribution.length {
-			break
-		}
-		relative.segments[itemPosition] = pathSegment{kind: pathSegmentIndex, index: itemIndex - contribution.offset}
-		break
-	}
 	if node, present := documentAtPathWithLookup(application.value, relative, lookup); present {
 		origin.Locations = []Location{publicLocation(node.Location)}
 	}

internal/engine/history_sequence_test.go

@@ -1,7 +1,9 @@
 package engine
 
 import (
+	"bytes"
 	"reflect"
+	"strings"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -137,6 +139,89 @@ func TestHistoryMergeSupportsSemanticIndexedPrefixes(t *testing.T) {
 	}
 }
 
+func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T) {
+	policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}}
+	composed, diagnostics := composeDocumentsWithPolicy(policy,
+		parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
+		parseCompositionDocument(t, "second", "records: [{id: second, token: private}]"),
+	)
+	if len(diagnostics) != 0 {
+		t.Fatalf("compose diagnostics = %#v", diagnostics)
+	}
+	path := Keys("records").Index(1).Key("token")
+	var metadata Metadata
+	finalizeMetadataPaths(&metadata, composed.root, composed.applications)
+	if !metadata.hasHistoricalPath(path) {
+		t.Fatalf("projected metadata omitted shifted path %s: %#v", path, metadata.historicalPaths)
+	}
+
+	inspection := &Inspection{
+		metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
+		input: composed.root, final: composed.root, applications: composed.applications,
+	}
+	inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
+		return reflect.DeepEqual(candidate, path)
+	}, metadata, composed.root, composed.applications)
+	explanation, ok := inspection.Explain(path)
+	if !ok || !explanation.Effective.Redacted {
+		t.Fatalf("shifted explanation = %#v, %v", explanation, ok)
+	}
+	var foundSecondOrigin bool
+	for _, change := range explanation.History {
+		for _, location := range change.Origin.Locations {
+			foundSecondOrigin = foundSecondOrigin || location.SourceID == "second"
+		}
+	}
+	if !foundSecondOrigin {
+		t.Fatalf("shifted history lost second source attribution: %#v", explanation.History)
+	}
+	var rendered bytes.Buffer
+	if err := inspection.WriteExplanation(&rendered, path); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(rendered.String(), `path: "/records/1/token"`) || strings.Contains(rendered.String(), "private") {
+		t.Fatalf("shifted explanation rendering is incomplete or unsafe: %s", rendered.String())
+	}
+}
+
+func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) {
+	policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}}
+	composed, diagnostics := composeDocumentsWithPolicy(policy,
+		parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"),
+		parseCompositionDocument(t, "second", "records: [{id: second, token: PRIVATE-RECORD-CANARY}]"),
+	)
+	if len(diagnostics) != 0 {
+		t.Fatalf("compose diagnostics = %#v", diagnostics)
+	}
+	applications := append([]historyApplication(nil), composed.applications...)
+	applications = append(applications, historyApplication{
+		layer: LayerOverlay,
+		value: parseCompositionDocument(t, "replacement", "records: [{id: replacement, token: visible}]"),
+	})
+	final := applications[len(applications)-1].value
+	path := Keys("records").Index(1).Key("token")
+	var metadata Metadata
+	finalizeMetadataPaths(&metadata, final, applications)
+	inspection := &Inspection{
+		metadata: metadata, baseline: &document.Node{Kind: document.Mapping},
+		input: final, final: final, applications: applications,
+	}
+	inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool {
+		return reflect.DeepEqual(candidate, path)
+	}, metadata, final, applications)
+	explanation, ok := inspection.Explain(path)
+	if !ok || explanation.Effective.Present {
+		t.Fatalf("replaced shifted record explanation = %#v, %v", explanation, ok)
+	}
+	var rendered bytes.Buffer
+	if err := inspection.WriteExplanation(&rendered, path); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(rendered.String(), "PRIVATE-RECORD-CANARY") || !strings.Contains(rendered.String(), "[REDACTED]") {
+		t.Fatalf("replaced sensitive history leaked or was not redacted: %s", rendered.String())
+	}
+}
+
 func mustParsePath(t *testing.T, value string) Path {
 	t.Helper()
 	path, err := ParsePath(value)

internal/engine/metadata.go

@@ -149,12 +149,9 @@ func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []his
 		walkDocument(tree, Path{}, add)
 	}
 	for _, application := range applications {
-		// Overrides have mapping-only prefixes; parsed tokens are keys in this layer.
+		walkHistoryApplication(application, func(_ Path, effective Path, _ *document.Node) {
-		prefix := Path{}
+			add(effective, nil)
-		for _, segment := range application.path.segments {
+		})
-			prefix = prefix.Key(segment.key)
-		}
-		walkDocument(application.value, prefix, add)
 	}
 	for _, operation := range m.inheritance {
 		if operation.control != nil {

internal/engine/secret.go

@@ -1,6 +1,10 @@
 package engine
 
-import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+import (
+	"slices"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
 
 type secretDependency struct {
 	target     Path
@@ -13,18 +17,40 @@ func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, me
 
 func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Path) bool, metadata Metadata, final *document.Node, applications []historyApplication, evidence []*sourceVersionEvidence, projections ...*document.Node) map[string]bool {
 	paths := make(map[string]Path)
+	candidateShapes := make(map[string][]Path)
+	var candidates []Path
+	addCandidate := func(path Path) {
+		pointer := path.String()
+		for _, prior := range candidateShapes[pointer] {
+			if slices.Equal(path.segments, prior.segments) {
+				return
+			}
+		}
+		cloned := path.clone()
+		candidateShapes[pointer] = append(candidateShapes[pointer], cloned)
+		candidates = append(candidates, cloned)
+	}
 	for _, application := range applications {
-		walkDocument(application.value, application.path, func(path Path, _ *document.Node) {
+		walkHistoryApplication(application, func(_ Path, path Path, _ *document.Node) {
+			if len(application.sequences) != 0 {
+				addCandidate(path)
+			} else if parsed, err := ParsePath(path.String()); err == nil {
+				addCandidate(parsed)
+			}
 			addSecretPath(paths, path.String())
 		})
 	}
 	var dependencies []secretDependency
 	for pointer, history := range metadata.history {
 		addSecretPath(paths, pointer)
+		if path, err := ParsePath(pointer); err == nil {
+			addCandidate(path)
+		}
 		for _, change := range history {
 			paths[change.Path.String()] = change.Path.clone()
 			for _, dependency := range change.Dependencies {
 				paths[dependency.String()] = dependency.clone()
+				addCandidate(dependency)
 				dependencies = append(dependencies, secretDependency{
 					target: change.Path.clone(), dependency: dependency.clone(),
 				})
@@ -36,8 +62,10 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 		if operation.control != nil {
 			control := operation.control.path
 			paths[control.String()] = control.clone()
+			addCandidate(control)
 			for _, contributor := range operation.controlOrigin.contributors {
 				paths[contributor.source.String()] = contributor.source.clone()
+				addCandidate(contributor.source)
 				inherited = append(inherited, secretDependency{target: control.clone(), dependency: contributor.source.clone()})
 			}
 			continue
@@ -57,6 +85,8 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 			destination := appendInheritancePath(operation.transfer.to, relative)
 			paths[pointer] = source
 			paths[destination.String()] = destination.clone()
+			addCandidate(source)
+			addCandidate(destination)
 			inherited = append(inherited, secretDependency{target: destination, dependency: source})
 		}
 	}
@@ -66,6 +96,9 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 		walkDocument(tree, Path{}, func(path Path, node *document.Node) {
 			pointer := path.String()
 			addSecretPath(paths, pointer)
+			if parsed, err := ParsePath(pointer); err == nil {
+				addCandidate(parsed)
+			}
 			if node.Sensitive {
 				secret[pointer] = true
 			}
@@ -83,7 +116,7 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 			secret[path.String()] = true
 		}
 	}
-	for _, path := range paths {
+	for _, path := range candidates {
 		classifyPath(path)
 	}
 	// Classify each original evidence shape before folding results into pointer
@@ -96,9 +129,13 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 		if !candidate.valid() {
 			candidate = item.path
 		}
-		classifyPath(addSecretEvidencePath(paths, item.destination, candidate))
+		resolved := addSecretEvidencePath(paths, item.destination, candidate)
+		addCandidate(resolved)
+		classifyPath(resolved)
 		for _, ancestor := range item.ancestors {
-			classifyPath(addSecretEvidencePath(paths, item.destination, ancestor.path))
+			resolved := addSecretEvidencePath(paths, item.destination, ancestor.path)
+			addCandidate(resolved)
+			classifyPath(resolved)
 		}
 	}
 	return closeSecretDependencies(paths, secret, dependencies, inherited)