diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index bff978f..189a879 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implement shared source version checks -Status: Stages 1–5 complete; direct underlay integration has not started. +Status: Stages 1–6 complete; selected overlay integration has not started. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. The [feature roadmap](source-version-checks.md) owns target behavior and @@ -379,6 +379,26 @@ No environmental limitation affected validation; checks ran on Linux/amd64. Validate: grouped/underlay/destination/version tests and everyday check. Done: direct and imported underlays follow the same checking policy. +Completion evidence: threaded the per-load version state through direct underlay +group acquisition and invoked the shared processor with contribution semantics +for each acquired underlay root. Its own processed payload/import declarations +now drive placement and recursive imports, while remaining placed payload is +still checked against the effective main-owned path before strict composition. +Warnings are accumulated through underlay groups; source records retain their +layer and group index, and the existing accounting, context, and reusable-source +paths remain in place. Added coverage for distinct declarations within one +strict group, warning retention across groups, rejected declarations beneath a +later main value, required omissions, empty groups and NoSource, mounted nested +headers/imports, and protected remaining payload. Existing default underlay +version-protection coverage remains passing. These checks passed with Go +`go1.26.8 linux/amd64`: + +- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'SourceVersion|Version|Import|Underlay|Overlay|Destination' -count=1` +- `GOTOOLCHAIN=local ./scripts/check.sh` +- `git diff --check` + +No environmental limitation affected validation; checks ran on Linux/amd64. + ## Stage 7 — Integrate selected overlays and final ownership checks 1. Process overlay headers after parse/structural validation, before remaining diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 250ad17..8ed0907 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -67,12 +67,20 @@ func loadImportedDocumentsTracked(source Source, context *FilesystemContext, mai } func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { + var processing *sourceVersionProcessingResult + if versionState != nil && versionState.mode != sourceVersionModeDefault { + processing = versionState.mainProcessing + } + return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing) +} + +func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) { if nativeContext(source, context) { access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context) if diagnostic != nil { return nil, sourceInfos(main.source), []Diagnostic{*diagnostic} } - return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState) + return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { @@ -87,7 +95,7 @@ func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemC if hostToken != nil { access.token = hostToken } - return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState) + return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) } func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) { @@ -99,6 +107,14 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath } func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) { + var processing *sourceVersionProcessingResult + if versionState != nil && versionState.mode != sourceVersionModeDefault { + processing = versionState.mainProcessing + } + return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing) +} + +func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) { if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) { _ = access.close() return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)} @@ -151,9 +167,9 @@ func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath var body *document.Node var declarations []importDeclaration var diagnostics []Diagnostic - if versionState != nil && versionState.mode != sourceVersionModeDefault && versionState.mainProcessing != nil { - body = versionState.mainProcessing.payload - declarations = versionState.mainProcessing.imports + if processing != nil { + body = processing.payload + declarations = processing.imports } else { body, declarations, diagnostics = splitImportDirectives(main.document) } diff --git a/internal/engine/load.go b/internal/engine/load.go index f6d2160..6758734 100644 --- a/internal/engine/load.go +++ b/internal/engine/load.go @@ -176,6 +176,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState) accumulator.sources = input.sources accumulator.add(versionState.diagnostics...) + reportedVersionDiagnostics := len(versionState.diagnostics) accumulator.add(diagnostics...) if hasErrors(diagnostics) { return nil, accumulator.loadError() @@ -200,6 +201,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo } else { input, diagnostics = assembleLayeredInput(request, input, accounting) } + accumulator.add(versionState.diagnostics[reportedVersionDiagnostics:]...) accumulator.sources = input.sources accumulator.add(diagnostics...) if len(diagnostics) != 0 { diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index c01627a..ba6ddba 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -26,6 +26,10 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], destinat } func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) { + return assembleBaseInputTrackedForRole(source, options, destination, accounting, visited, hostToken, versionState, sourceVersionMain) +} + +func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, role sourceVersionRole) (assembledInput, []Diagnostic) { limits := defaultImportLimits() parseLimits := defaultParseLimits() parseLimits.source.remaining = limits.bytes - accounting.bytes @@ -47,25 +51,32 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T } return input, diagnostics } + var versionProcessing *sourceVersionProcessingResult if versionState != nil && versionState.mode != sourceVersionModeDefault { var sourceID string if parsed.source != nil { sourceID = parsed.source.ID } - processed, failures := versionState.process(parsed.document, sourceVersionMain, destination, sourceID) + processed, failures := versionState.process(parsed.document, role, destination, sourceID) if len(failures) != 0 { return input, failures } - versionState.mainProcessing = &processed + versionProcessing = &processed + if role == sourceVersionMain { + versionState.mainProcessing = &processed + } } if options.Imports { - input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithVersion(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState) + input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing) if len(diagnostics) != 0 { return input, diagnostics } input.document = input.composition.root } else { input.sources = sourceInfos(parsed.source) + if versionProcessing != nil { + input.document = versionProcessing.payload + } if options.Filesystem != nil && !activeOverlay(options.Overlay) { if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil { return input, []Diagnostic{*diagnostic} diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index becef88..1fd20e4 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -79,7 +79,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) { groups := make([]*composedDocument, 0, len(request.underlays)) for index, group := range request.underlays { - composition, sources, contexts, diagnostics := acquireUnderlayGroup(group, index+1, request.protectionVersion(), accounting) + composition, sources, contexts, diagnostics := acquireUnderlayGroupWithVersion(group, index+1, request.protectionVersion(), accounting, input.versionState) input.sources = append(input.sources, sources...) mergeResourceContexts(&input.contexts, contexts) if len(diagnostics) != 0 { @@ -149,6 +149,10 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a } func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { + return acquireUnderlayGroupWithVersion(group, index, rule, accounting, nil) +} + +func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { initialBytes, initialNodes := accounting.bytes, accounting.nodes visited := make(map[resourceIdentity]importRoute) hostToken := &namespaceToken{} @@ -157,7 +161,7 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account var contexts map[string]ResourceContext for _, contribution := range group.inputs { options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}} - acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, contribution.destination, accounting, visited, hostToken) + acquired, diagnostics := assembleBaseInputTrackedForRole(contribution.source, options, contribution.destination, accounting, visited, hostToken, versionState, sourceVersionContribution) for _, source := range acquired.sources { source.Layer = LayerUnderlay source.UnderlayIndex = index diff --git a/internal/engine/source_version_test.go b/internal/engine/source_version_test.go index eaaeaf8..4aa05fd 100644 --- a/internal/engine/source_version_test.go +++ b/internal/engine/source_version_test.go @@ -2,6 +2,7 @@ package engine import ( "errors" + "path/filepath" "testing" "gitea.maximumdirect.net/eric/ConfigKit/internal/document" @@ -754,6 +755,225 @@ configkit: }) } +func TestSourceVersionUnderlaysCheckDistinctMountedDeclarations(t *testing.T) { + t.Parallel() + + var checked []string + definition := Definition[map[string]any]{Version: &VersionRule{ + Path: Keys("meta", "version"), + Check: func(version Version, validation *Validation) error { + checked = append(checked, version.Text) + if version.Text == "peer-a" || version.Text == "peer-c" { + validation.Warning(Path{}, "accepted with advisory") + } + return nil + }, + }} + main := Input{ + Source: Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n")), + Destination: Keys("service"), + } + first := Input{ + Source: Bytes("peer-a", []byte("meta:\n version: peer-a\nvalues: {first: one}\n")), + Destination: Keys("defaults", "first"), + } + second := Input{ + Source: Bytes("peer-b", []byte("meta:\n version: peer-b\nvalues: {second: two}\n")), + Destination: Keys("defaults", "second"), + } + third := Input{ + Source: Bytes("peer-c", []byte("meta:\n version: peer-c\nvalues: {third: three}\n")), + Destination: Keys("defaults", "third"), + } + loaded, err := LoadLayered(main, testSecretBinding(), LayeredOptions[map[string]any]{ + Definition: definition, SourceVersionMode: sourceVersionModeRequired, + Underlays: []Underlay{{Inputs: []Input{first, second}}, {Inputs: []Input{third}}}, + }) + if err != nil { + t.Fatalf("LoadLayered: %v", err) + } + if len(checked) != 4 { + t.Fatalf("checked declarations = %#v, want main and all underlays", checked) + } + for _, want := range []string{"main-v", "peer-a", "peer-b", "peer-c"} { + found := false + for _, got := range checked { + found = found || got == want + } + if !found { + t.Errorf("checked declarations %#v do not include %q", checked, want) + } + } + if len(loaded.Inspection.Diagnostics()) != 2 || loaded.Inspection.Diagnostics()[0].Code != CodeVersionWarning || loaded.Inspection.Diagnostics()[1].Code != CodeVersionWarning { + t.Fatalf("accepted underlay warning was not retained: %#v", loaded.Inspection.Diagnostics()) + } + sources := loaded.Inspection.Sources() + if len(sources) != 4 || sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 || sources[3].Layer != LayerUnderlay || sources[3].UnderlayIndex != 2 { + t.Fatalf("underlay source records = %#v", sources) + } + defaults, ok := loaded.Config["defaults"].(map[string]any) + if !ok { + t.Fatalf("defaults = %#v", loaded.Config["defaults"]) + } + firstValue, ok := defaults["first"].(map[string]any) + if !ok || firstValue["values"] == nil { + t.Fatalf("first mounted contribution = %#v", defaults["first"]) + } + if meta, ok := firstValue["meta"].(map[string]any); !ok || len(meta) != 0 { + t.Fatalf("consumed nested declaration did not preserve its empty ancestor: %#v", firstValue["meta"]) + } +} + +func TestSourceVersionUnderlaysRejectDeclarationsHiddenByMain(t *testing.T) { + t.Parallel() + + var checked []string + loaded, err := LoadLayered(Input{Source: Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n"))}, testSecretBinding(), LayeredOptions[map[string]any]{ + Definition: Definition[map[string]any]{Version: &VersionRule{ + Path: Keys("meta", "version"), + Check: func(version Version, _ *Validation) error { + checked = append(checked, version.Text) + if version.Text == "rejected-peer" { + return errors.New("peer declaration is not accepted") + } + return nil + }, + }}, + SourceVersionMode: sourceVersionModeIfPresent, + Underlays: []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte("meta: {version: rejected-peer}\nservice: shadowed\n"))}}}}, + }) + if err == nil || loaded != nil { + t.Fatalf("LoadLayered = %#v, %v, want hidden underlay rejection", loaded, err) + } + loadErr, ok := err.(*LoadError) + if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionHookFailed { + t.Fatalf("underlay rejection diagnostics = %#v", err) + } + if len(checked) != 2 || checked[0] != "main-v" || checked[1] != "rejected-peer" { + t.Fatalf("checked declarations = %#v", checked) + } +} + +func TestSourceVersionUnderlaysRequireActualHeadersButSkipAbsentSources(t *testing.T) { + t.Parallel() + + newOptions := func(underlays []Underlay) LayeredOptions[map[string]any] { + return LayeredOptions[map[string]any]{ + Definition: Definition[map[string]any]{Version: &VersionRule{ + Path: Keys("meta", "version"), + Check: func(Version, *Validation) error { return nil }, + }}, + SourceVersionMode: sourceVersionModeRequired, + Underlays: underlays, + } + } + main := Input{Source: Bytes("main", []byte("meta: {version: main-v}\n"))} + + t.Run("required absence", func(t *testing.T) { + loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{Inputs: []Input{{Source: Bytes("missing", []byte("value: present\n"))}}}})) + if err == nil || loaded != nil { + t.Fatalf("LoadLayered = %#v, %v, want missing underlay declaration", loaded, err) + } + loadErr, ok := err.(*LoadError) + if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionError { + t.Fatalf("missing underlay diagnostics = %#v", err) + } + }) + + t.Run("empty group and NoSource", func(t *testing.T) { + loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{}, {Inputs: []Input{{Source: NoSource()}}}})) + if err != nil { + t.Fatalf("LoadLayered with absent contributions: %v", err) + } + if sources := loaded.Inspection.Sources(); len(sources) != 1 { + t.Fatalf("empty groups and NoSource created source records: %#v", sources) + } + }) +} + +func TestSourceVersionUnderlayImportsKeepLocalHeadersAndMountedDestinations(t *testing.T) { + t.Parallel() + + directory := t.TempDir() + underlayPath := writeImportFile(t, directory, "underlay.yml", `meta: {version: peer-root} +configkit: + imports: + - path: nested.yml + destination: [nested] +root: retained +`) + writeImportFile(t, directory, "nested.yml", "meta: {version: peer-nested}\nname: nested\n") + var checked []string + loaded, err := LoadLayered(Input{ + Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"), + }, testSecretBinding(), LayeredOptions[map[string]any]{ + Definition: Definition[map[string]any]{Version: &VersionRule{ + Path: Keys("meta", "version"), + Check: func(version Version, _ *Validation) error { + checked = append(checked, version.Text) + return nil + }, + }}, + SourceVersionMode: sourceVersionModeRequired, + Underlays: []Underlay{{Inputs: []Input{{ + Source: File(underlayPath), Imports: true, Destination: Keys("defaults"), + }}}}, + }) + if err != nil { + t.Fatalf("LoadLayered: %v", err) + } + if len(checked) != 3 { + t.Fatalf("checked source declarations = %#v", checked) + } + for _, want := range []string{"main-v", "peer-root", "peer-nested"} { + found := false + for _, got := range checked { + found = found || got == want + } + if !found { + t.Errorf("checked source declarations %#v do not include %q", checked, want) + } + } + sources := loaded.Inspection.Sources() + if len(sources) != 3 || sources[1].Path != filepath.Clean(underlayPath) || sources[2].Destination.String() != "/defaults/nested" { + t.Fatalf("underlay and nested import records = %#v", sources) + } + if sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 { + t.Fatalf("underlay and nested import tiers = %#v", sources) + } + defaults, ok := loaded.Config["defaults"].(map[string]any) + if !ok || defaults["root"] != "retained" { + t.Fatalf("mounted underlay root = %#v", loaded.Config["defaults"]) + } + nested, ok := defaults["nested"].(map[string]any) + if !ok || nested["name"] != "nested" { + t.Fatalf("mounted nested import = %#v", defaults["nested"]) + } +} + +func TestSourceVersionUnderlaysProtectRemainingMountedPayload(t *testing.T) { + t.Parallel() + + loaded, err := LoadLayered(Input{ + Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"), + }, testSecretBinding(), LayeredOptions[map[string]any]{ + Definition: Definition[map[string]any]{Version: &VersionRule{ + Path: Keys("meta", "version"), Check: func(Version, *Validation) error { return nil }, + }}, + SourceVersionMode: sourceVersionModeRequired, + Underlays: []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte( + "meta: {version: accepted}\nservice: {meta: {version: payload}}\n", + ))}}}}, + }) + if err == nil || loaded != nil { + t.Fatalf("LoadLayered = %#v, %v, want protected remaining payload rejection", loaded, err) + } + loadErr, ok := err.(*LoadError) + if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeUnderlayVersion { + t.Fatalf("remaining payload diagnostics = %#v", err) + } +} + func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T) { t.Parallel()