From 66dced082358eef843005e3c8b790891fc96af89 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 16 Oct 2026 15:44:44 +0000 Subject: [PATCH] Project history and sensitivity paths --- docs/roadmap/implementation.md | 18 ++++- internal/engine/history.go | 82 +++++++++++++++++------ internal/engine/history_sequence_test.go | 85 ++++++++++++++++++++++++ internal/engine/metadata.go | 9 +-- internal/engine/secret.go | 47 +++++++++++-- 5 files changed, 207 insertions(+), 34 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 117d4e2..9b70a5a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implementation Plan: Sequence Placement and Composition -Status: implementation in progress; stages 1–4 completed. +Status: implementation in progress; stages 1–5 completed. ## Purpose and authority @@ -372,6 +372,22 @@ Validation: metadata/provenance/history/explanation/secret tests, `go test -race ./internal/engine`, and `./scripts/check.sh`. Completion: the private registered composer can feed existing inspection safely. +Completion evidence: history applications now expose one source-local/effective +path traversal used by metadata enumeration, explanation replay, origin lookup, +and secret classification. Appended records and nested descendants use assigned +indexes while retained nodes and source locations remain local. Secret +classification evaluates projected historical shapes before folding sensitivity +by pointer, preserving numeric key/index distinctions and redacting replaced +records. Tests cover shifted metadata/history/rendering, origin attribution, and +redaction after replacement. These checks passed with Go `go1.26.9 linux/amd64`: + +- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'Metadata|Provenance|History|Explain|Secret' -count=1` +- `GOTOOLCHAIN=local GOPROXY=off go test -race ./internal/engine` +- `GOTOOLCHAIN=local ./scripts/check.sh` +- `git diff --check` + +No environmental limitation affected validation; checks ran on Linux/amd64. + ## Stage 6 — Refactor acquisition into composition scopes without new behavior Scope: `imports.go`, `load_input.go`, `load_request.go`, and their handoffs. diff --git a/internal/engine/history.go b/internal/engine/history.go index d2b9b43..1c31353 100644 --- a/internal/engine/history.go +++ b/internal/engine/history.go @@ -66,6 +66,61 @@ func cloneHistoryApplications(values []historyApplication) []historyApplication return cloned } +// walkHistoryApplication visits a retained source tree with both its local +// semantic path and its effective projected path. Sequence fragments keep +// their original tree indexes, while the effective path uses the assigned +// range recorded by strict composition. +func walkHistoryApplication(application historyApplication, visit func(local, effective Path, node *document.Node)) { + if application.value == nil { + return + } + var walk func(*document.Node, Path, Path) + walk = func(node *document.Node, local, effective Path) { + if node == nil { + return + } + visit(local, appendDestination(application.path, effective), node) + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + walk(entry.Value, local.Key(entry.Key), effective.Key(entry.Key)) + } + case document.Sequence: + contribution, projected := sequenceHistoryAt(application.sequences, local) + for index, child := range node.Sequence { + effectiveIndex := index + if projected { + effectiveIndex += contribution.offset + } + walk(child, local.Index(index), effective.Index(effectiveIndex)) + } + } + } + walk(application.value, Path{}, Path{}) +} + +// historyApplicationLocalPath translates an effective projected path back to +// the retained source tree, including indexes shifted by enclosing appends. +func historyApplicationLocalPath(application historyApplication, effective Path) (Path, bool) { + relative, ok := relativeHistoryPath(application.path, effective) + if !ok { + return Path{}, false + } + local := Path{} + for _, segment := range relative.segments { + if contribution, found := sequenceHistoryAt(application.sequences, local); found { + index, valid := schemaIndex(segment) + if !valid || segment.kind == pathSegmentKey || index < contribution.offset || index >= contribution.offset+contribution.length { + return Path{}, false + } + local = local.Index(index - contribution.offset) + continue + } + local.segments = append(local.segments, segment) + } + return local, true +} + func applicationOperationPath(application historyApplication, query Path, lookup *documentLookup) (Path, bool) { if !application.path.valid() || application.value == nil { return Path{}, false @@ -78,14 +133,13 @@ func applicationOperationPath(application historyApplication, query Path, lookup } sequencePathMatched := false for _, contribution := range application.sequences { - if !pathContains(contribution.path, query) && !pathContains(query, contribution.path) { + container := appendDestination(application.path, contribution.path) + if !pathContains(container, query) && !pathContains(query, container) { continue } sequencePathMatched = true - if pathContains(contribution.path, query) && len(query.segments) > len(contribution.path.segments) { - segment := query.segments[len(contribution.path.segments)] - itemIndex, ok := schemaIndex(segment) - if !ok || itemIndex < contribution.offset || itemIndex >= contribution.offset+contribution.length { + if pathContains(container, query) && len(query.segments) > len(container.segments) { + if _, ok := historyApplicationLocalPath(application, query); !ok { continue } } @@ -115,26 +169,10 @@ func historyApplicationOrigin(application historyApplication, operationPath Path return cloneOrigin(application.origin) } origin := Origin{Layer: application.layer} - relative, ok := relativeHistoryPath(application.path, operationPath) + relative, ok := historyApplicationLocalPath(application, operationPath) if !ok { return origin } - for _, contribution := range application.sequences { - if !pathContains(contribution.path, operationPath) { - continue - } - relative, ok = relativeHistoryPath(Path{}, operationPath) - if !ok || len(relative.segments) < len(contribution.path.segments)+1 { - break - } - itemPosition := len(contribution.path.segments) - itemIndex, valid := schemaIndex(relative.segments[itemPosition]) - if !valid || itemIndex < contribution.offset || itemIndex >= contribution.offset+contribution.length { - break - } - relative.segments[itemPosition] = pathSegment{kind: pathSegmentIndex, index: itemIndex - contribution.offset} - break - } if node, present := documentAtPathWithLookup(application.value, relative, lookup); present { origin.Locations = []Location{publicLocation(node.Location)} } diff --git a/internal/engine/history_sequence_test.go b/internal/engine/history_sequence_test.go index d667169..37fdda6 100644 --- a/internal/engine/history_sequence_test.go +++ b/internal/engine/history_sequence_test.go @@ -1,7 +1,9 @@ package engine import ( + "bytes" "reflect" + "strings" "testing" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" @@ -137,6 +139,89 @@ func TestHistoryMergeSupportsSemanticIndexedPrefixes(t *testing.T) { } } +func TestProjectedSequenceHistoryAgreesAcrossMetadataAndExplanation(t *testing.T) { + policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}} + composed, diagnostics := composeDocumentsWithPolicy(policy, + parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"), + parseCompositionDocument(t, "second", "records: [{id: second, token: private}]"), + ) + if len(diagnostics) != 0 { + t.Fatalf("compose diagnostics = %#v", diagnostics) + } + path := Keys("records").Index(1).Key("token") + var metadata Metadata + finalizeMetadataPaths(&metadata, composed.root, composed.applications) + if !metadata.hasHistoricalPath(path) { + t.Fatalf("projected metadata omitted shifted path %s: %#v", path, metadata.historicalPaths) + } + + inspection := &Inspection{ + metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, + input: composed.root, final: composed.root, applications: composed.applications, + } + inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { + return reflect.DeepEqual(candidate, path) + }, metadata, composed.root, composed.applications) + explanation, ok := inspection.Explain(path) + if !ok || !explanation.Effective.Redacted { + t.Fatalf("shifted explanation = %#v, %v", explanation, ok) + } + var foundSecondOrigin bool + for _, change := range explanation.History { + for _, location := range change.Origin.Locations { + foundSecondOrigin = foundSecondOrigin || location.SourceID == "second" + } + } + if !foundSecondOrigin { + t.Fatalf("shifted history lost second source attribution: %#v", explanation.History) + } + var rendered bytes.Buffer + if err := inspection.WriteExplanation(&rendered, path); err != nil { + t.Fatal(err) + } + if !strings.Contains(rendered.String(), `path: "/records/1/token"`) || strings.Contains(rendered.String(), "private") { + t.Fatalf("shifted explanation rendering is incomplete or unsafe: %s", rendered.String()) + } +} + +func TestReplacedProjectedSensitiveRecordRemainsRedacted(t *testing.T) { + policy := sequencePolicy{identityFields: map[string]string{"/records": "id"}} + composed, diagnostics := composeDocumentsWithPolicy(policy, + parseCompositionDocument(t, "first", "records: [{id: first, token: public}]"), + parseCompositionDocument(t, "second", "records: [{id: second, token: PRIVATE-RECORD-CANARY}]"), + ) + if len(diagnostics) != 0 { + t.Fatalf("compose diagnostics = %#v", diagnostics) + } + applications := append([]historyApplication(nil), composed.applications...) + applications = append(applications, historyApplication{ + layer: LayerOverlay, + value: parseCompositionDocument(t, "replacement", "records: [{id: replacement, token: visible}]"), + }) + final := applications[len(applications)-1].value + path := Keys("records").Index(1).Key("token") + var metadata Metadata + finalizeMetadataPaths(&metadata, final, applications) + inspection := &Inspection{ + metadata: metadata, baseline: &document.Node{Kind: document.Mapping}, + input: final, final: final, applications: applications, + } + inspection.secrets = classifySecretPaths(nil, func(candidate Path) bool { + return reflect.DeepEqual(candidate, path) + }, metadata, final, applications) + explanation, ok := inspection.Explain(path) + if !ok || explanation.Effective.Present { + t.Fatalf("replaced shifted record explanation = %#v, %v", explanation, ok) + } + var rendered bytes.Buffer + if err := inspection.WriteExplanation(&rendered, path); err != nil { + t.Fatal(err) + } + if strings.Contains(rendered.String(), "PRIVATE-RECORD-CANARY") || !strings.Contains(rendered.String(), "[REDACTED]") { + t.Fatalf("replaced sensitive history leaked or was not redacted: %s", rendered.String()) + } +} + func mustParsePath(t *testing.T, value string) Path { t.Helper() path, err := ParsePath(value) diff --git a/internal/engine/metadata.go b/internal/engine/metadata.go index 2b98b9f..c9cf937 100644 --- a/internal/engine/metadata.go +++ b/internal/engine/metadata.go @@ -149,12 +149,9 @@ func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []his walkDocument(tree, Path{}, add) } for _, application := range applications { - // Overrides have mapping-only prefixes; parsed tokens are keys in this layer. - prefix := Path{} - for _, segment := range application.path.segments { - prefix = prefix.Key(segment.key) - } - walkDocument(application.value, prefix, add) + walkHistoryApplication(application, func(_ Path, effective Path, _ *document.Node) { + add(effective, nil) + }) } for _, operation := range m.inheritance { if operation.control != nil { diff --git a/internal/engine/secret.go b/internal/engine/secret.go index ff15fbb..18f609d 100644 --- a/internal/engine/secret.go +++ b/internal/engine/secret.go @@ -1,6 +1,10 @@ package engine -import "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +import ( + "slices" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) type secretDependency struct { target Path @@ -13,18 +17,40 @@ func classifySecretPaths(schema *schemaDescriptor, predicate func(Path) bool, me func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Path) bool, metadata Metadata, final *document.Node, applications []historyApplication, evidence []*sourceVersionEvidence, projections ...*document.Node) map[string]bool { paths := make(map[string]Path) + candidateShapes := make(map[string][]Path) + var candidates []Path + addCandidate := func(path Path) { + pointer := path.String() + for _, prior := range candidateShapes[pointer] { + if slices.Equal(path.segments, prior.segments) { + return + } + } + cloned := path.clone() + candidateShapes[pointer] = append(candidateShapes[pointer], cloned) + candidates = append(candidates, cloned) + } for _, application := range applications { - walkDocument(application.value, application.path, func(path Path, _ *document.Node) { + walkHistoryApplication(application, func(_ Path, path Path, _ *document.Node) { + if len(application.sequences) != 0 { + addCandidate(path) + } else if parsed, err := ParsePath(path.String()); err == nil { + addCandidate(parsed) + } addSecretPath(paths, path.String()) }) } var dependencies []secretDependency for pointer, history := range metadata.history { addSecretPath(paths, pointer) + if path, err := ParsePath(pointer); err == nil { + addCandidate(path) + } for _, change := range history { paths[change.Path.String()] = change.Path.clone() for _, dependency := range change.Dependencies { paths[dependency.String()] = dependency.clone() + addCandidate(dependency) dependencies = append(dependencies, secretDependency{ target: change.Path.clone(), dependency: dependency.clone(), }) @@ -36,8 +62,10 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa if operation.control != nil { control := operation.control.path paths[control.String()] = control.clone() + addCandidate(control) for _, contributor := range operation.controlOrigin.contributors { paths[contributor.source.String()] = contributor.source.clone() + addCandidate(contributor.source) inherited = append(inherited, secretDependency{target: control.clone(), dependency: contributor.source.clone()}) } continue @@ -57,6 +85,8 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa destination := appendInheritancePath(operation.transfer.to, relative) paths[pointer] = source paths[destination.String()] = destination.clone() + addCandidate(source) + addCandidate(destination) inherited = append(inherited, secretDependency{target: destination, dependency: source}) } } @@ -66,6 +96,9 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa walkDocument(tree, Path{}, func(path Path, node *document.Node) { pointer := path.String() addSecretPath(paths, pointer) + if parsed, err := ParsePath(pointer); err == nil { + addCandidate(parsed) + } if node.Sensitive { secret[pointer] = true } @@ -83,7 +116,7 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa secret[path.String()] = true } } - for _, path := range paths { + for _, path := range candidates { classifyPath(path) } // Classify each original evidence shape before folding results into pointer @@ -96,9 +129,13 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa if !candidate.valid() { candidate = item.path } - classifyPath(addSecretEvidencePath(paths, item.destination, candidate)) + resolved := addSecretEvidencePath(paths, item.destination, candidate) + addCandidate(resolved) + classifyPath(resolved) for _, ancestor := range item.ancestors { - classifyPath(addSecretEvidencePath(paths, item.destination, ancestor.path)) + resolved := addSecretEvidencePath(paths, item.destination, ancestor.path) + addCandidate(resolved) + classifyPath(resolved) } } return closeSecretDependencies(paths, secret, dependencies, inherited)