eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Adopt ConfigKit overlays for pipeline selection

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 15:24:03 +00:00
parent e27c49e946
commit 83eaa960ec
38 changed files with 394 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/config/artifact_catalog.go
View File

@@ -93,7 +93,7 @@ func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Pat
seen := map[string]struct{}{}
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerProfile {
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerOverlay {
continue
}
for _, location := range change.Origin.Locations {

14
internal/config/effective_digest.go
View File

@@ -14,21 +14,21 @@ const (
effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n"
)
// PipelineProfileProvenance identifies the profile selected while resolving a
// PipelineOverlayProvenance identifies the overlay selected while resolving a
// pipeline. It contains only non-secret selection metadata.
type PipelineProfileProvenance struct {
type PipelineOverlayProvenance struct {
Name string
Source string
OverlayPath string
}
// SelectedPipelineProfile reports the profile used to resolve cfg, if any.
func SelectedPipelineProfile(cfg *PipelineConfig) (*PipelineProfileProvenance, bool) {
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil {
// SelectedPipelineOverlay reports the overlay used to resolve cfg, if any.
func SelectedPipelineOverlay(cfg *PipelineConfig) (*PipelineOverlayProvenance, bool) {
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedOverlay == nil {
return nil, false
}
selection := cfg.resolution.selectedProfile
return &PipelineProfileProvenance{Name: selection.name, Source: selection.source, OverlayPath: selection.overlayPath}, true
selection := cfg.resolution.selectedOverlay
return &PipelineOverlayProvenance{Name: selection.name, Source: selection.source, OverlayPath: selection.overlayPath}, true
}
// EffectivePipelineDigest reports the deterministic secret-free digest for a

4
internal/config/effective_pipeline.go
View File

@@ -401,8 +401,8 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
if source == cfg.resolution.rootPath {
return "root"
}
if cfg.resolution.selectedProfile != nil && source == cfg.resolution.selectedProfile.overlayPath {
return "profile"
if cfg.resolution.selectedOverlay != nil && source == cfg.resolution.selectedOverlay.overlayPath {
return "overlay"
}
for _, imported := range cfg.resolution.imports {
if source == imported {

26
internal/config/load.go
View File

@@ -15,14 +15,14 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return LoadPipelineWithOptions(path, PipelineLoadOptions{})
}
// PipelineLoadOptions carries an optional explicit profile selection. Nil,
// PipelineLoadOptions carries an optional explicit overlay selection. Nil,
// empty, and the exact name "default" select the base configuration.
type PipelineLoadOptions struct {
Profile *string
Overlay *string
}
// LoadPipelineWithOptions loads pipeline configuration with strict field
// checking and optional named-profile selection.
// checking and optional named-overlay selection.
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
cfg, err := loadPipelineWithConfigKit(path, opts)
if err != nil {
@@ -31,20 +31,20 @@ func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineCo
return finalizeLoadedPipeline(path, cfg)
}
// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
if leftProfile == rightProfile {
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
// LoadPipelineOverlayPair resolves two overlays as independent ConfigKit loads.
func LoadPipelineOverlayPair(path, leftOverlay, rightOverlay string) (*PipelineConfig, *PipelineConfig, error) {
if leftOverlay == rightOverlay {
return nil, nil, fmt.Errorf("load pipeline overlays: left and right overlay selections must differ")
}
leftOptions := PipelineLoadOptions{Profile: &leftProfile}
rightOptions := PipelineLoadOptions{Profile: &rightProfile}
leftOptions := PipelineLoadOptions{Overlay: &leftOverlay}
rightOptions := PipelineLoadOptions{Overlay: &rightOverlay}
left, err := LoadPipelineWithOptions(path, leftOptions)
if err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
return nil, nil, fmt.Errorf("load left pipeline overlay: %w", err)
}
right, err := LoadPipelineWithOptions(path, rightOptions)
if err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
return nil, nil, fmt.Errorf("load right pipeline overlay: %w", err)
}
return left, right, nil
}
@@ -76,7 +76,7 @@ func LoadSession(path string) (*SessionConfig, error) {
type SessionLoadOptions struct {
SessionID string
PreviousSessionID string
Profile *string
Overlay *string
}
// LoadSessionWithOptions loads session configuration from a YAML file with
@@ -143,7 +143,7 @@ func Load(pipelinePath string, paths ...string) (*Config, error) {
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
// session configuration with expected session identity checks.
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Profile: sessionOpts.Profile})
pipelineCfg, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Overlay: sessionOpts.Overlay})
if err != nil {
return nil, err
}

10
internal/config/load_validate_test.go
View File

@@ -358,7 +358,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
t.Run(profile.name, func(t *testing.T) {
options := PipelineLoadOptions{}
if profile.selection != "" {
options.Profile = &profile.selection
options.Overlay = &profile.selection
}
pipeline, err := LoadPipelineWithOptions(pipelinePath, options)
if err != nil {
@@ -381,12 +381,12 @@ func TestExamplesLoadAndValidate(t *testing.T) {
if cfg.Pipeline.Audita.LLMAPIKey != nil || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
t.Fatalf("split bundle must remain offline and secret-free: audita=%#v storage=%#v", cfg.Pipeline.Audita, cfg.Pipeline.Storage)
}
selectedProfile, selected := SelectedPipelineProfile(cfg.Pipeline)
selectedOverlay, selected := SelectedPipelineOverlay(cfg.Pipeline)
if profile.selection == "" && selected {
t.Fatalf("production base unexpectedly selected profile %#v", selectedProfile)
t.Fatalf("production base unexpectedly selected overlay %#v", selectedOverlay)
}
if profile.selection != "" && (!selected || selectedProfile.Name != profile.selection) {
t.Fatalf("selected profile = %#v, want %q", selectedProfile, profile.selection)
if profile.selection != "" && (!selected || selectedOverlay.Name != profile.selection) {
t.Fatalf("selected overlay = %#v, want %q", selectedOverlay, profile.selection)
}
for _, key := range []string{
"session_recap",

24
internal/config/pipeline_configkit.go
View File

@@ -16,7 +16,7 @@ type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
selectedOverlay *pipelineOverlaySelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
@@ -29,7 +29,7 @@ type pipelineResolutionMetadata struct {
configkitSources []configkit.SourceInfo
}
type pipelineProfileSelection struct {
type pipelineOverlaySelection struct {
name string
source string
overlayPath string
@@ -41,8 +41,8 @@ type pipelineFieldOwnership struct {
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
profile := selectedPipelineProfileName(opts)
result, err := loadPipelineConfigKitResult(path, profile)
overlay := selectedPipelineOverlayName(opts)
result, err := loadPipelineConfigKitResult(path, overlay)
if err != nil {
return nil, describePipelineLoadError(err)
}
@@ -55,10 +55,10 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline
return &cfg, nil
}
func loadPipelineConfigKitResult(path, profile string) (*configkit.Result[PipelineConfig], error) {
func loadPipelineConfigKitResult(path, overlay string) (*configkit.Result[PipelineConfig], error) {
return configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
Imports: true,
Profile: profile,
Overlay: overlay,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: resolvePipelineConfig,
@@ -104,11 +104,11 @@ func describeConfigKitLoadError(kind string, err error) error {
return fmt.Errorf("load %s config: strict decode failed or validation failed: %w: %s", kind, err, strings.TrimSpace(diagnostics.String()))
}
func selectedPipelineProfileName(opts PipelineLoadOptions) string {
if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" {
func selectedPipelineOverlayName(opts PipelineLoadOptions) string {
if opts.Overlay == nil || *opts.Overlay == "" || *opts.Overlay == "default" {
return ""
}
return *opts.Profile
return *opts.Overlay
}
func pipelineConfigDefaults() PipelineConfig {
@@ -282,8 +282,8 @@ func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result
} else if source.Layer == configkit.LayerBase {
resolved.imports = append(resolved.imports, path)
}
if source.Layer == configkit.LayerProfile {
resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path}
if source.Layer == configkit.LayerOverlay {
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
}
}
detailed, err := pipelineMetadataPaths(cfg)
@@ -347,7 +347,7 @@ func rejectDuplicateBaseDefinitions(metadata configkit.Metadata, sources []confi
pathsByID[source.ID] = source.Path
}
// Enumerate retained declarations, not just surviving effective keys. Loading
// the base separately would also resolve credentials removed by the profile.
// the base separately would also resolve credentials removed by the overlay.
// ConfigKit returns canonical JSON Pointers in deterministic order.
for _, path := range metadata.HistoricalPaths() {
parts := strings.Split(path.String(), "/")

71
internal/config/pipeline_configkit_test.go
View File

@@ -8,7 +8,7 @@ import (
"testing"
)
func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) {
func TestPipelineConfigKitImportsOverlayAndBaseSelection(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
@@ -24,33 +24,62 @@ whisperx:
enabled: false
output_path: artifacts/recap.md
`)
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n")
writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n")
writePipelineFixture(t, dir, "overlays/testing.yml", "workspace:\n root: /srv/testing\n")
writePipelineFixture(t, dir, "overlays/identical.yml", "workspace:\n root: /srv/base\n")
writePipelineFixture(t, dir, "overlays/unselected.yml", "unknown: [\n")
root := filepath.Join(dir, "pipeline.yml")
var baseDigest string
for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} {
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile})
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: profile})
if err != nil {
t.Fatalf("load base selection %#v: %v", profile, err)
}
if cfg.Workspace.Root != "/srv/base" {
t.Fatalf("base workspace = %q", cfg.Workspace.Root)
}
if baseDigest == "" {
baseDigest = EffectivePipelineDigest(cfg)
} else if EffectivePipelineDigest(cfg) != baseDigest {
t.Fatal("equivalent base selections changed effective digest")
}
}
identical := "identical"
unchanged, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &identical})
if err != nil {
t.Fatal(err)
}
if EffectivePipelineDigest(unchanged) != baseDigest {
t.Fatal("equivalent overlay changed effective digest")
}
selected := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected})
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 {
t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg))
}
profile, ok := SelectedPipelineProfile(cfg)
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") {
profile, ok := SelectedPipelineOverlay(cfg)
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "overlays", "testing.yml") {
t.Fatalf("selected profile = %#v, %v", profile, ok)
}
}
func TestPipelineOverlayDoesNotDiscoverLegacyDirectory(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", "configkit:\n imports:\n - path: profiles/shared.yml\n")
writePipelineFixture(t, dir, "profiles/shared.yml", "workspace:\n root: /srv/base\n")
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/old\n")
if base, err := LoadPipeline(filepath.Join(dir, "pipeline.yml")); err != nil || base.Workspace.Root != "/srv/base" {
t.Fatalf("explicit import from profiles directory = %#v, %v", base, err)
}
selected := "testing"
if _, err := LoadPipelineWithOptions(filepath.Join(dir, "pipeline.yml"), PipelineLoadOptions{Overlay: &selected}); err == nil || !strings.Contains(err.Error(), "overlay.missing") {
t.Fatalf("legacy selected file result = %v, want missing overlay", err)
}
}
func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) {
for _, test := range []struct {
name, first, second, clearedContainer string
@@ -69,14 +98,14 @@ whisperx:
`)
writePipelineFixture(t, dir, "first.yml", test.first)
writePipelineFixture(t, dir, "second.yml", test.second)
writePipelineFixture(t, dir, "profiles/clear-container.yml", test.clearedContainer)
writePipelineFixture(t, dir, "profiles/clear-scriptorium.yml", "scriptorium: null\n")
writePipelineFixture(t, dir, "overlays/clear-container.yml", test.clearedContainer)
writePipelineFixture(t, dir, "overlays/clear-scriptorium.yml", "scriptorium: null\n")
for _, profile := range []string{"", "clear-container", "clear-scriptorium"} {
t.Run(profile, func(t *testing.T) {
options := PipelineLoadOptions{}
if profile != "" {
options.Profile = &profile
options.Overlay = &profile
}
_, err := LoadPipelineWithOptions(filepath.Join(dir, "pipeline.yml"), options)
if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") ||
@@ -89,7 +118,7 @@ whisperx:
}
}
func TestPipelineProfileDoesNotRequireUnusedBaseCredentials(t *testing.T) {
func TestPipelineOverlayDoesNotRequireUnusedBaseCredentials(t *testing.T) {
const production = "NARRATIO_PROFILE_UNUSED_PROD_KEY"
const testingKey = "NARRATIO_PROFILE_TEST_KEY"
t.Setenv(production, "")
@@ -101,9 +130,9 @@ func TestPipelineProfileDoesNotRequireUnusedBaseCredentials(t *testing.T) {
t.Run(credential, func(t *testing.T) {
dir := t.TempDir()
root := writePipelineFixture(t, dir, "pipeline.yml", "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\naudita:\n llm_api_key: "+production+"\n")
writePipelineFixture(t, dir, "profiles/testing.yml", "audita:\n llm_api_key: "+credential+"\n")
writePipelineFixture(t, dir, "overlays/testing.yml", "audita:\n llm_api_key: "+credential+"\n")
profile := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &profile})
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &profile})
if err != nil {
t.Fatal(err)
}
@@ -130,9 +159,9 @@ func TestNotariusCredentialProfileReplacementAndProvenance(t *testing.T) {
t.Run(credential, func(t *testing.T) {
dir := t.TempDir()
root := writePipelineFixture(t, dir, "pipeline.yml", "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+baseKey+"\n")
overlay := writePipelineFixture(t, dir, "profiles/testing.yml", "notarius:\n llm_api_key: "+credential+"\n")
overlay := writePipelineFixture(t, dir, "overlays/testing.yml", "notarius:\n llm_api_key: "+credential+"\n")
profile := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &profile})
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &profile})
if err != nil {
t.Fatal(err)
}
@@ -187,8 +216,8 @@ func TestNotariusConfigProfileComposition(t *testing.T) {
root := writePipelineFixture(t, dir, "pipeline.yml", rootYAML)
options := PipelineLoadOptions{}
if test.overlayName != "" {
writePipelineFixture(t, dir, "profiles/"+test.overlayName+".yml", test.overlay)
options.Profile = &test.overlayName
writePipelineFixture(t, dir, "overlays/"+test.overlayName+".yml", test.overlay)
options.Overlay = &test.overlayName
}
cfg, err := LoadPipelineWithOptions(root, options)
if err != nil {
@@ -238,9 +267,9 @@ func TestNotariusCacheControlProfileReplacementAndNullReset(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
root := writePipelineFixture(t, dir, "pipeline.yml", "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n resume: false\n chunk_cache: bypass\n")
overlay := writePipelineFixture(t, dir, "profiles/testing.yml", test.overlay)
overlay := writePipelineFixture(t, dir, "overlays/testing.yml", test.overlay)
profile := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &profile})
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &profile})
if err != nil {
t.Fatal(err)
}
@@ -282,9 +311,9 @@ whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "artifact.yml", "scriptorium:\n artifacts:\n recap:\n prompt_id: base\n output_path: artifacts/recap.md\n")
writePipelineFixture(t, dir, "profiles/testing.yml", "scriptorium:\n artifacts:\n recap:\n prompt_id: testing\n")
writePipelineFixture(t, dir, "overlays/testing.yml", "scriptorium:\n artifacts:\n recap:\n prompt_id: testing\n")
profile := "testing"
cfg, err := LoadPipelineWithOptions(filepath.Join(dir, "pipeline.yml"), PipelineLoadOptions{Profile: &profile})
cfg, err := LoadPipelineWithOptions(filepath.Join(dir, "pipeline.yml"), PipelineLoadOptions{Overlay: &profile})
if err != nil {
t.Fatal(err)
}