eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enforce the retained history node limit

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-25 22:04:54 +00:00
parent 4fcff5f453
commit 12d350a292
9 changed files with 395 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/roadmap/implementation.md
View File

@@ -416,6 +416,13 @@ Focused check when buildable: `go test ./internal/engine -run 'Load|History|Expl
Finding: completes **D01**.
Owners: `internal/engine/history.go`, `load.go`, `load_result.go`, history/lifecycle tests.
Status: **complete** (D01). Prior completed commits: `2736ddf`, `deb9aa5`,
`ae3408a`, `ea27f17`, `08a8c25`, `877c497`, `8aba43d`, `7bf2f0c`,
`0f9ecf1`, `119b42c`, `247a12b`, `4fcff5f`. Validation passed:
`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'History|Explain|Inheritance|Override|Resolution|Load' -count=1`,
`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
and `git diff --check`.
1. Use one explicit root visitor on the prepared inspection, feeding a
pointer-aware `retainedNodeCounter` fresh for this publication, not the earlier
transient expansion counter. Include baseline, input, final, pre-CLI,

12
internal/engine/explain_test.go
View File

@@ -308,12 +308,18 @@ func TestSecretsBootstrapLoadAndExplanationRemainSeparateAndSafe(t *testing.T) {
func TestHistoryNodeBudgetCountsSharedTreesOnce(t *testing.T) {
leaf := &document.Node{Kind: document.String, Text: "value"}
root := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{Key: "value", Value: leaf}}}
applications := []historyApplication{{layer: LayerBase, value: root}}
if !historyNodesWithin(2, root, root, root, applications, nil) {
inspection := &Inspection{
baseline: root, input: root, final: root,
applications: []historyApplication{{layer: LayerBase, value: root}},
}
counter := newRetainedNodeCounter(2)
if !addInspectionHistory(counter, inspection) || counter.count != 2 {
t.Fatal("structurally shared history was counted more than once")
}
other := document.Clone(root)
if historyNodesWithin(3, root, other, root, applications, nil) {
inspection.input = other
counter = newRetainedNodeCounter(3)
if addInspectionHistory(counter, inspection) {
t.Fatal("distinct retained nodes did not consume the budget")
}
}

143
internal/engine/history.go
View File

@@ -40,6 +40,86 @@ func (c *retainedNodeCounter) add(root *document.Node) bool {
return true
}
func (c *retainedNodeCounter) canAddClones(roots []*document.Node, copies int) bool {
if c == nil || c.limit < 0 {
return true
}
remaining := c.limit - c.count
if remaining < 0 {
return false
}
for copyIndex := 0; copyIndex < copies; copyIndex++ {
for _, root := range roots {
count, within := documentCloneNodeCount(root, remaining)
if !within {
return false
}
remaining -= count
}
}
return true
}
func documentCloneNodeCount(root *document.Node, limit int) (int, bool) {
if root == nil {
return 0, true
}
count := 0
stack := []*document.Node{root}
for len(stack) != 0 {
node := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if node == nil {
continue
}
count++
if count > limit {
return 0, false
}
for _, entry := range node.Mapping {
stack = append(stack, entry.Value)
}
stack = append(stack, node.Sequence...)
}
return count, true
}
func addInspectionHistory(counter *retainedNodeCounter, inspection *Inspection) bool {
if inspection == nil {
return false
}
for _, root := range []*document.Node{inspection.baseline, inspection.input, inspection.final, inspection.preCLI} {
if !counter.add(root) {
return false
}
}
for _, application := range inspection.applications {
if !counter.add(application.value) {
return false
}
}
for _, application := range inspection.postDecodeApps {
if !counter.add(application.value) {
return false
}
}
for _, snapshots := range [][]resolutionSnapshot{inspection.resolution, inspection.resolutionReplay} {
for _, snapshot := range snapshots {
if !counter.add(snapshot.before) || !counter.add(snapshot.after) {
return false
}
}
}
for _, operations := range [][]inheritanceOperation{inspection.inheritance, inspection.metadata.inheritance} {
for _, operation := range operations {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
return false
}
}
}
return true
}
// History applications borrow immutable policy-validated input trees. Metadata
// records answer presence and origin queries; replay reconstructs value changes.
// Replay reconciles with canonical decoding, applies supplied CLI contributions,
@@ -499,66 +579,3 @@ func replaceHistoryPath(root **document.Node, path Path, replacement *document.N
}
}
}
func historyNodesWithinLimit(baseline, input, final *document.Node, applications []historyApplication, resolutions []resolutionSnapshot, inherited ...*inheritanceScope) bool {
return historyNodesWithin(historyNodeLimit, baseline, input, final, applications, resolutions, inherited...)
}
func historyNodesWithin(limit int, baseline, input, final *document.Node, applications []historyApplication, resolutions []resolutionSnapshot, inherited ...*inheritanceScope) bool {
seen := make(map[*document.Node]bool)
count := 0
var visit func(*document.Node) bool
visit = func(node *document.Node) bool {
if node == nil || seen[node] {
return true
}
seen[node] = true
count++
if count > limit {
return false
}
for _, entry := range node.Mapping {
if !visit(entry.Value) {
return false
}
}
for _, child := range node.Sequence {
if !visit(child) {
return false
}
}
return true
}
if !visit(baseline) || !visit(input) || !visit(final) {
return false
}
for _, application := range applications {
if !visit(application.value) {
return false
}
}
for _, resolution := range resolutions {
if !visit(resolution.before) || !visit(resolution.after) {
return false
}
}
for _, expansion := range inherited {
if expansion == nil {
continue
}
if !visit(expansion.root) {
return false
}
for _, operation := range expansion.operations {
if !visit(operation.parent) || !visit(operation.effective) {
return false
}
}
for _, replacement := range expansion.replacements {
if !visit(replacement.before) || !visit(replacement.after) {
return false
}
}
}
return true
}

210
internal/engine/history_budget_test.go Normal file
View File

@@ -0,0 +1,210 @@
package engine
import (
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
type historyBudgetSimpleConfig struct {
Value string
}
type historyBudgetRecord struct {
Value string
Credential Secret
}
type historyBudgetLayeredConfig struct {
Parent historyBudgetRecord
Child historyBudgetRecord
}
func retainedInspectionNodeCountForTest(inspection *Inspection) int {
seen := make(map[*document.Node]struct{})
stack := []*document.Node{inspection.baseline, inspection.input, inspection.final, inspection.preCLI}
for _, application := range inspection.applications {
stack = append(stack, application.value)
}
for _, application := range inspection.postDecodeApps {
stack = append(stack, application.value)
}
for _, snapshot := range inspection.resolution {
stack = append(stack, snapshot.before, snapshot.after)
}
for _, snapshot := range inspection.resolutionReplay {
stack = append(stack, snapshot.before, snapshot.after)
}
for _, operation := range inspection.inheritance {
stack = append(stack, operation.parent, operation.effective)
}
for _, operation := range inspection.metadata.inheritance {
stack = append(stack, operation.parent, operation.effective)
}
for len(stack) != 0 {
last := len(stack) - 1
node := stack[last]
stack = stack[:last]
if node == nil {
continue
}
if _, exists := seen[node]; exists {
continue
}
seen[node] = struct{}{}
for _, entry := range node.Mapping {
stack = append(stack, entry.Value)
}
stack = append(stack, node.Sequence...)
}
return len(seen)
}
func TestLoadHistoryBudgetUsesExactRetainedNodeCount(t *testing.T) {
load := func(historyLimit int, validations *int) (*Loaded[historyBudgetSimpleConfig], error) {
return loadWithSecretLookupLimit(
Bytes("main", []byte("value: retained\n")),
testSecretBinding(),
func(string) (string, bool) { return "", false },
historyLimit,
Options[historyBudgetSimpleConfig]{Definition: Definition[historyBudgetSimpleConfig]{
Validate: func(*historyBudgetSimpleConfig, *Validation) { *validations++ },
}},
)
}
var baselineValidations int
baseline, err := load(10000, &baselineValidations)
if err != nil {
t.Fatalf("baseline load: %v", err)
}
want := retainedInspectionNodeCountForTest(baseline.Inspection)
if want == 0 {
t.Fatal("independent retained-tree count was empty")
}
var exactValidations int
loaded, err := load(want, &exactValidations)
if err != nil || loaded == nil {
t.Fatalf("load at exact retained-node count %d: result=%v error=%v", want, loaded, err)
}
if got := retainedInspectionNodeCountForTest(loaded.Inspection); got != want {
t.Fatalf("retained node count = %d, want %d", got, want)
}
if exactValidations != 1 {
t.Fatalf("validation calls at exact count = %d, want 1", exactValidations)
}
var rejectedValidations int
loaded, err = load(want-1, &rejectedValidations)
if loaded != nil || !hasHistoryLimitDiagnostic(t, err) {
t.Fatalf("load below retained-node count: result=%v error=%v", loaded, err)
}
if rejectedValidations != 0 {
t.Fatalf("validation ran before history rejection: %d calls", rejectedValidations)
}
}
func TestLayeredHistoryBudgetRejectsBeforeSecretsAndValidation(t *testing.T) {
type callbackCounts struct {
lookups int
validations int
}
makeRequest := func(counts *callbackCounts) loadRequest[historyBudgetLayeredConfig] {
return loadRequest[historyBudgetLayeredConfig]{
main: loadInput{source: Bytes("main", []byte("child:\n extends: parent\n"))},
underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("underlay", []byte(
"parent:\n value: inherited\n credential: HISTORY_BUDGET_SECRET\n",
))}}}},
overrides: []Override{{Path: Keys("child", "value"), Value: "cli", Flag: "--child-value"}},
definition: Definition[historyBudgetLayeredConfig]{
Resolve: func(configuration *historyBudgetLayeredConfig, resolution *Resolution) error {
configuration.Child.Value += "-resolved"
resolution.Record(Keys("child", "value"), ChangeNormalization, "normalize child")
return nil
},
Validate: func(*historyBudgetLayeredConfig, *Validation) { counts.validations++ },
},
}
}
lookup := func(counts *callbackCounts) secretLookup {
return func(name string) (string, bool) {
counts.lookups++
if name != "HISTORY_BUDGET_SECRET" {
t.Errorf("secret lookup name = %q", name)
}
return "resolved-secret", true
}
}
baselineCounts := &callbackCounts{}
baseline, err := loadGroupedWithSecretLookupLimit(makeRequest(baselineCounts), testSecretBinding(), lookup(baselineCounts), 10000)
if err != nil {
t.Fatalf("baseline grouped load: %v", err)
}
want := retainedInspectionNodeCountForTest(baseline.Inspection)
if want == 0 {
t.Fatal("independent retained-tree count was empty")
}
if len(baseline.Inspection.inheritance) == 0 || len(baseline.Inspection.metadata.inheritance) == 0 {
t.Fatal("layered load did not retain both inheritance evidence copies")
}
exactCounts := &callbackCounts{}
loaded, err := loadGroupedWithSecretLookupLimit(makeRequest(exactCounts), testSecretBinding(), lookup(exactCounts), want)
if err != nil || loaded == nil {
t.Fatalf("grouped load at exact retained-node count %d: result=%v error=%v", want, loaded, err)
}
if got := retainedInspectionNodeCountForTest(loaded.Inspection); got != want {
t.Fatalf("grouped retained node count = %d, want %d", got, want)
}
if loaded.Config.Child.Value != "cli-resolved" || exactCounts.lookups != 1 || exactCounts.validations != 1 {
t.Fatalf("grouped lifecycle result=%#v callbacks=%#v", loaded.Config, exactCounts)
}
rejectedCounts := &callbackCounts{}
loaded, err = loadGroupedWithSecretLookupLimit(makeRequest(rejectedCounts), testSecretBinding(), lookup(rejectedCounts), want-1)
if loaded != nil || !hasHistoryLimitDiagnostic(t, err) {
t.Fatalf("grouped load below retained-node count: result=%v error=%v", loaded, err)
}
if rejectedCounts.lookups != 0 || rejectedCounts.validations != 0 {
t.Fatalf("callbacks ran before history rejection: %#v", rejectedCounts)
}
}
func TestPreparedInheritanceEvidenceOwnsDistinctTrees(t *testing.T) {
root := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{Key: "value", Value: &document.Node{Kind: document.String, Text: "shared"}}}}
resolved := resolvedLoad[historyBudgetSimpleConfig]{
inheritance: &inheritanceScope{operations: []inheritanceOperation{{parent: root, effective: root}}},
}
counter := newRetainedNodeCounter(8)
inspection, ok := prepareInspectionHistory(resolved, counter)
if !ok {
t.Fatal("preparation rejected the exact size of the distinct retained copies")
}
if len(inspection.inheritance) != 1 || len(inspection.metadata.inheritance) != 1 {
t.Fatalf("inheritance copies = inspection %#v, metadata %#v", inspection.inheritance, inspection.metadata.inheritance)
}
inspectionOperation := inspection.inheritance[0]
metadataOperation := inspection.metadata.inheritance[0]
if inspectionOperation.parent == metadataOperation.parent || inspectionOperation.effective == metadataOperation.effective || counter.count != 8 {
t.Fatalf("separate inheritance copies were not counted independently: inspection=(%p, %p) metadata=(%p, %p) count=%d", inspectionOperation.parent, inspectionOperation.effective, metadataOperation.parent, metadataOperation.effective, counter.count)
}
}
func hasHistoryLimitDiagnostic(t *testing.T, err error) bool {
t.Helper()
if err == nil {
return false
}
loadErr, ok := err.(*LoadError)
if !ok {
t.Fatalf("error type = %T, want *LoadError", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code == CodeExplainLimitExceeded {
return true
}
}
return false
}

1
internal/engine/inheritance_history.go
View File

@@ -74,7 +74,6 @@ func applyInheritanceMetadata(metadata *Metadata, raw *document.Node, rawOrigins
})
applyOverlayMetadata(&sourceView, overlays)
metadata.history = sourceView.history
metadata.inheritance = cloneInheritanceOperations(expansion.operations)
}
func historyWithInheritance(path Path, stored []Change, operations []inheritanceOperation) []Change {

16
internal/engine/inheritance_history_test.go
View File

@@ -124,6 +124,7 @@ func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocume
}
metadata := metadataFromDecoded(decoded)
applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded)
metadata.inheritance = cloneInheritanceOperations(expanded.operations)
finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root)
return &Inspection{
metadata: metadata, baseline: baseline, input: expanded.root, preCLI: expanded.root,
@@ -388,15 +389,18 @@ func TestInheritanceHistoryPrecedesApplicationResolution(t *testing.T) {
func TestHistoryNodeBudgetCountsSharedInheritanceEvidenceOnce(t *testing.T) {
shared := &document.Node{Kind: document.String, Text: "shared"}
extra := &document.Node{Kind: document.String, Text: "extra"}
expansion := &inheritanceScope{
root: shared,
operations: []inheritanceOperation{{parent: shared, effective: shared}},
replacements: []inheritanceReplacement{{before: shared, after: extra}},
inspection := &Inspection{
inheritance: []inheritanceOperation{{parent: shared, effective: shared}},
metadata: Metadata{inheritance: []inheritanceOperation{{parent: shared, effective: extra}}},
}
if !historyNodesWithin(2, nil, nil, nil, nil, nil, expansion) ||
historyNodesWithin(1, nil, nil, nil, nil, nil, expansion) {
counter := newRetainedNodeCounter(2)
if !addInspectionHistory(counter, inspection) || counter.count != 2 {
t.Fatal("shared evidence node accounting is incorrect")
}
counter = newRetainedNodeCounter(1)
if addInspectionHistory(counter, inspection) {
t.Fatal("distinct inheritance evidence did not consume the budget")
}
}
func TestInheritanceExpansionEnforcesRetainedHistoryBudget(t *testing.T) {

17
internal/engine/load.go
View File

@@ -148,6 +148,10 @@ func Load[T any](source Source, secrets SecretBinding, supplied ...Options[T]) (
}
func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup secretLookup, supplied ...Options[T]) (*Loaded[T], error) {
return loadWithSecretLookupLimit(source, secrets, lookup, historyNodeLimit, supplied...)
}
func loadWithSecretLookupLimit[T any](source Source, secrets SecretBinding, lookup secretLookup, historyLimit int, supplied ...Options[T]) (*Loaded[T], error) {
if len(supplied) > 1 {
return nil, (&diagnosticAccumulator{diagnostics: []Diagnostic{optionDiagnostic(CodeOptionsInvalid, "Load accepts at most one options value")}}).loadError()
}
@@ -174,10 +178,10 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se
sourceVersionMode: options.SourceVersionMode,
effectiveVersion: effectiveSourceVersionRuleForPlacement(options.Definition.Version, input.destination, options.SourceVersionMode, input.placementMode),
}
return executeLoadRequest(request, secrets, lookup)
return executeLoadRequest(request, secrets, lookup, historyLimit)
}
func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup, historyLimit int) (*Loaded[T], error) {
accumulator := diagnosticAccumulator{}
accounting := &importAccounting{}
policy := acquisitionPolicy{
@@ -341,11 +345,16 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
if hasErrors(diagnostics) {
return nil, accumulator.loadError()
}
inspection := prepareInspectionHistory(resolved)
counter := newRetainedNodeCounter(historyLimit)
inspection, admitted := prepareInspectionHistory(resolved, counter)
if !admitted {
accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"})
return nil, accumulator.loadError()
}
historyApplications := make([]historyApplication, 0, len(inspection.applications)+len(inspection.postDecodeApps))
historyApplications = append(historyApplications, inspection.applications...)
historyApplications = append(historyApplications, inspection.postDecodeApps...)
if !historyNodesWithinLimit(inspection.baseline, inspection.input, inspection.final, historyApplications, inspection.resolution, input.inheritance) {
if !addInspectionHistory(counter, inspection) {
accumulator.add(Diagnostic{Code: CodeExplainLimitExceeded, Severity: SeverityError, Message: "retained configuration history exceeds the node limit"})
return nil, accumulator.loadError()
}

6
internal/engine/load_request.go
View File

@@ -48,6 +48,10 @@ type acquiredUnderlayGroup struct {
}
func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
return loadGroupedWithSecretLookupLimit(request, secrets, lookup, historyNodeLimit)
}
func loadGroupedWithSecretLookupLimit[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup, historyLimit int) (*Loaded[T], error) {
request.ordinary = false
defaults, policy, diagnostics := loadSettings(request.importDefaults, request.definition, request.sourceVersionMode)
request.importDefaults = defaults
@@ -76,7 +80,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
if len(diagnostics) != 0 {
return nil, (&diagnosticAccumulator{diagnostics: canonicalDiagnostics(diagnostics)}).loadError()
}
return executeLoadRequest(request, secrets, lookup)
return executeLoadRequest(request, secrets, lookup, historyLimit)
}
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, policy acquisitionPolicy, state *acquisitionState) (assembledInput, []Diagnostic) {

67
internal/engine/load_result.go
View File

@@ -99,21 +99,76 @@ func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, m
return detached.Interface().(T), nil
}
func prepareInspectionHistory[T any](resolved resolvedLoad[T]) *Inspection {
return &Inspection{
metadata: resolved.metadata,
func prepareInspectionHistory[T any](resolved resolvedLoad[T], counter *retainedNodeCounter) (*Inspection, bool) {
metadata := resolved.metadata
metadata.inheritance = nil
inspection := &Inspection{
metadata: metadata,
baseline: resolved.decoded.baseline,
input: resolved.decoded.final,
inputOrigins: cloneOriginIndex(resolved.decoded.origins),
preCLI: resolved.preCLI,
preCLIOrigins: cloneOrigins(resolved.preCLIOrigins),
postDecodeApps: cloneHistoryApplications(resolved.postDecodeApps),
final: resolved.final,
resolution: resolved.snapshots,
resolutionReplay: coalescedResolutionSnapshots(resolved.snapshots),
applications: cloneHistoryApplications(resolved.applications),
inheritance: inheritanceOperations(resolved.inheritance),
}
if !addInspectionHistory(counter, inspection) {
return nil, false
}
var ok bool
inspection.applications, ok = cloneHistoryApplicationsWithinLimit(resolved.applications, counter)
if !ok {
return nil, false
}
inspection.postDecodeApps, ok = cloneHistoryApplicationsWithinLimit(resolved.postDecodeApps, counter)
if !ok {
return nil, false
}
if resolved.inheritance != nil {
roots := inheritanceOperationDocumentRoots(resolved.inheritance.operations)
if !counter.canAddClones(roots, 2) {
return nil, false
}
inspection.metadata.inheritance = cloneInheritanceOperations(resolved.inheritance.operations)
if !addInheritanceOperationDocuments(counter, inspection.metadata.inheritance) {
return nil, false
}
inspection.inheritance = cloneInheritanceOperations(resolved.inheritance.operations)
if !addInheritanceOperationDocuments(counter, inspection.inheritance) {
return nil, false
}
}
if !addInspectionHistory(counter, inspection) {
return nil, false
}
return inspection, true
}
func cloneHistoryApplicationsWithinLimit(values []historyApplication, counter *retainedNodeCounter) ([]historyApplication, bool) {
for _, value := range values {
if !counter.add(value.value) {
return nil, false
}
}
return cloneHistoryApplications(values), true
}
func inheritanceOperationDocumentRoots(operations []inheritanceOperation) []*document.Node {
roots := make([]*document.Node, 0, len(operations)*2)
for _, operation := range operations {
roots = append(roots, operation.parent, operation.effective)
}
return roots
}
func addInheritanceOperationDocuments(counter *retainedNodeCounter, operations []inheritanceOperation) bool {
for _, operation := range operations {
if !counter.add(operation.parent) || !counter.add(operation.effective) {
return false
}
}
return true
}
func publishLoadResult[T any](schema *schemaDescriptor, definition Definition[T], resolved resolvedLoad[T], inspection *Inspection, applications []historyApplication, references map[string]string, accumulator *diagnosticAccumulator) (*Loaded[T], error) {