From f302488075a83d2bfe58e2718d1983720baf6a82 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 30 Aug 2026 13:25:02 +0000 Subject: [PATCH] Add named pipeline profile composition --- docs/config.md | 41 +- docs/internal/configuration.md | 38 +- docs/roadmap/implementation.md | 2 +- internal/config/effective_digest.go | 69 ++++ internal/config/load.go | 21 +- internal/config/pipeline_composition.go | 257 +++++++++++- internal/config/pipeline_composition_test.go | 6 +- internal/config/pipeline_profiles_test.go | 392 +++++++++++++++++++ 8 files changed, 802 insertions(+), 24 deletions(-) create mode 100644 internal/config/effective_digest.go create mode 100644 internal/config/pipeline_profiles_test.go diff --git a/docs/config.md b/docs/config.md index 06c3f49..05defa1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -59,6 +59,10 @@ remote state with an unsafe legacy identity must be migrated before use. root-only `composition.imports` list. Imported files contribute fields to one logical pipeline document; they do not override fields supplied by the root or another import. +- A root pipeline may declare named profiles. Exactly one profile is selected + by an option-aware caller or by `composition.default_profile`; a caller's + explicit selection takes precedence. Declaring profiles without either form + of selection is an error. - Configured timeout and retry-delay durations must be positive. An omitted artifact timeout continues to inherit its configured Scriptorium timeout. - Session files must be concrete; unresolved `{{ ... }}` placeholders fail load. @@ -72,15 +76,22 @@ remote state with an unsafe legacy identity must be migrated before use. - local (`audio_dir` or `audio_files`), or - S3 (`audio_s3.prefix`). -### Pipeline imports +### Pipeline composition -Large pipeline configurations may be split into explicitly named fragments: +Large pipeline configurations may be split into explicitly named fragments and +may declare one overlay per selectable profile: ```yaml composition: imports: - config/storage.yml - config/integrations.yaml + default_profile: production + profiles: + production: + overlay: profiles/production.yml + testing: + overlay: profiles/testing.yml campaigns: root: /usr/local/share/narratio/campaigns @@ -100,6 +111,30 @@ name the full field path and every source that claimed it. The assembled YAML is then decoded against the normal strict pipeline schema and defaults are applied once. +Profile names are case-sensitive, non-empty, trimmed, and cannot contain +control characters. If `profiles` is present, it must contain at least one +entry and every entry must contain only an `overlay` path. An explicit profile +selection overrides `default_profile`; unknown and explicitly empty selections +fail. Narratio never selects the first profile implicitly and does not read a +profile selection from the environment. + +Every declared overlay is resolved relative to the root pipeline directory and +must satisfy the same confined regular-YAML-file rules as an import. Narratio +parses every declared overlay even when it is not selected, then applies only +the selected one. Maps merge recursively, overlay scalars replace base scalars, +and overlay lists replace base lists completely. Explicit `false`, zero, empty +lists, and empty maps remain meaningful. YAML null cannot delete a value, and +kind changes are rejected. Profiles cannot inherit from or stack with other +profiles, and overlays cannot import files or declare profiles. + +After composition, Narratio strictly decodes the result, applies centralized +defaults once, resolves ordinary paths, and computes a deterministic effective +configuration digest. The digest represents the normalized, secret-free +runtime pipeline mapping; it excludes composition declarations, source +provenance, profile identity, and raw environment secret values. Equivalent +effective mappings therefore have the same digest regardless of how fields are +split among the root and imports. + An imported field has the same meaning it would have in a monolithic root pipeline. In particular, ordinary relative pipeline paths continue to resolve from the root pipeline directory, not from the importing fragment's directory. @@ -192,6 +227,8 @@ Rules: | Field | Type | Required | Default / Rule | | --- | --- | --- | --- | | `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only | +| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile | +| `composition.profiles..overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory | | `pipeline.workspace.root` | string | No | `/var/lib/narratio` | | `pipeline.workspace.cleanup_after_publish` | bool | No | `false` | | `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` | diff --git a/docs/internal/configuration.md b/docs/internal/configuration.md index bffda3e..c705d92 100644 --- a/docs/internal/configuration.md +++ b/docs/internal/configuration.md @@ -12,17 +12,25 @@ pipeline-loading boundary implemented by `internal/config`. source names, full field paths, node kinds, declaration order, and explicit zero, false, empty-map, and empty-list values. 2. Remove the root-only `composition` envelope and validate its explicit - `imports` list. + `imports`, `default_profile`, and named `profiles` declarations. A load + option retains the difference between omitted and explicitly empty profile + selection. 3. Open each import relative to the root pipeline directory through the confined regular-file boundary. Imports must use a `.yml` or `.yaml` extension and cannot traverse, use symlinks, repeat a file, import the root, or contain another composition envelope. -4. Additively merge the root body and imports. Distinct map leaves compose; +4. Resolve and structurally parse every declared profile overlay through the + same confined regular-file boundary. Missing or malformed unselected + overlays fail the load. Overlays cannot contain a composition envelope. +5. Additively merge the root body and imports. Distinct map leaves compose; repeated scalar or list paths and node-kind disagreements are conflicts. -5. Emit deterministic canonical YAML and strictly decode it into +6. Select exactly one declared profile from an explicit option or the default, + then recursively merge its overlay. Overlay leaves replace base leaves, + lists are atomic replacements, and null or kind changes fail. +7. Emit deterministic canonical YAML and strictly decode it into `PipelineConfig`. -6. Apply pipeline defaults once, then resolve ordinary relative pipeline paths - from the root pipeline file. +8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from + the root pipeline file, and digest the normalized effective mapping. This ordering preserves monolithic configuration behavior. Moving a field to an imported fragment changes its source ownership, not its path base, default, @@ -36,13 +44,27 @@ claiming source so operators can repair the split without repeatedly rediscovering additional conflicts. The loaded pipeline retains private runtime metadata for the absolute root -path, ordered imports, contributing sources, and field ownership. This metadata -does not participate in YAML decoding or alter the public configuration model. +path, ordered imports, selected profile name and selection source, selected +overlay, contributing sources, effective digest, and leaf ownership. Base +leaves retain their root/import owners, replaced leaves belong to the selected +overlay, and centrally supplied values use the synthetic `default` owner. This +metadata does not participate in YAML decoding or alter the public +configuration model. + +The effective digest is SHA-256 over deterministic canonical YAML produced from +the defaulted and path-resolved `PipelineConfig`. Because composition and +resolution metadata are private, the digest excludes source layout, profile +name, and ownership. Configuration stores environment variable names rather +than resolving raw credentials, so raw secret values are neither loaded nor +hashed. `recomputePipelineEffectiveDigest` is the single package-owned refresh +point for later runtime expansion. ## Test Surfaces `composition_test.go` protects the presence and merge algebra independently of the public schema. `pipeline_composition_test.go` exercises explicit imports, confinement, conflicts, strict decoding, metadata, and root-relative path -behavior through `LoadPipeline`. Other configuration tests continue to protect +behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection, +all-overlay validation, overlay behavior, provenance, option propagation, and +effective-digest stability. Other configuration tests continue to protect defaults and validation after assembly. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 84bb622..72a139c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -445,7 +445,7 @@ weakening extract validation or analyze's per-artifact granularity. ## Stage 7 — Named Profile Composition And Effective Digest -**Status: Pending** +**Status: Completed** ### Goal diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go new file mode 100644 index 0000000..d2038fd --- /dev/null +++ b/internal/config/effective_digest.go @@ -0,0 +1,69 @@ +package config + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + + "gopkg.in/yaml.v3" +) + +const pipelineDefaultOwnershipSource = "default" + +func finalizePipelineResolution(cfg *PipelineConfig) error { + if cfg == nil || cfg.resolution == nil { + return fmt.Errorf("pipeline resolution metadata is required") + } + declared := make(map[string][]string, len(cfg.resolution.ownership)) + for _, ownership := range cfg.resolution.ownership { + declared[ownership.path] = append([]string(nil), ownership.sources...) + } + data, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("serialize normalized effective pipeline: %w", err) + } + document, err := parseCompositionBytes("normalized effective pipeline", data) + if err != nil { + return err + } + records, err := document.semanticRecords() + if err != nil { + return err + } + ownership := make([]pipelineFieldOwnership, 0, len(records)) + for _, record := range records { + sources := declared[record.Path] + if len(sources) == 0 { + sources = []string{pipelineDefaultOwnershipSource} + } + ownership = append(ownership, pipelineFieldOwnership{ + path: record.Path, sources: append([]string(nil), sources...), + }) + } + cfg.resolution.ownership = ownership + return recomputePipelineEffectiveDigest(cfg) +} + +// recomputePipelineEffectiveDigest is the single package-owned hook for +// refreshing provenance after later resolution expands concrete pipeline +// values. Composition declarations and runtime provenance are not serialized. +func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { + if cfg == nil || cfg.resolution == nil { + return fmt.Errorf("pipeline resolution metadata is required") + } + data, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("serialize normalized effective pipeline: %w", err) + } + document, err := parseCompositionBytes("normalized effective pipeline", data) + if err != nil { + return err + } + canonical, err := document.canonicalDigestInput() + if err != nil { + return err + } + digest := sha256.Sum256(canonical) + cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:]) + return nil +} diff --git a/internal/config/load.go b/internal/config/load.go index 0f46237..70d49cb 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -14,7 +14,20 @@ import ( // LoadPipeline loads pipeline configuration from a YAML file with strict field checking. func LoadPipeline(path string) (*PipelineConfig, error) { - cfg, err := loadComposedPipeline(path) + return LoadPipelineWithOptions(path, PipelineLoadOptions{}) +} + +// PipelineLoadOptions carries an optional explicit profile selection. A nil +// Profile means the caller omitted selection; a non-nil empty value is an +// explicit invalid selection. +type PipelineLoadOptions struct { + Profile *string +} + +// LoadPipelineWithOptions loads pipeline configuration with strict field +// checking and optional named-profile selection. +func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { + cfg, err := loadComposedPipeline(path, opts) if err != nil { return nil, fmt.Errorf("load pipeline config: %w", err) } @@ -22,6 +35,9 @@ func LoadPipeline(path string) (*PipelineConfig, error) { if err := resolveNotariusPaths(cfg, path); err != nil { return nil, fmt.Errorf("load pipeline config: %w", err) } + if err := finalizePipelineResolution(cfg); err != nil { + return nil, fmt.Errorf("load pipeline config: %w", err) + } return cfg, nil } @@ -43,6 +59,7 @@ func LoadSession(path string) (*SessionConfig, error) { type SessionLoadOptions struct { SessionID string PreviousSessionID string + Profile *string } // LoadSessionWithOptions loads session configuration from a YAML file with @@ -141,7 +158,7 @@ func Load(pipelinePath string, paths ...string) (*Config, error) { // LoadWithSessionOptions loads and resolves combined pipeline, campaign, and // session configuration with expected session identity checks. func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) { - pipelineCfg, err := LoadPipeline(pipelinePath) + pipelineCfg, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Profile: sessionOpts.Profile}) if err != nil { return nil, err } diff --git a/internal/config/pipeline_composition.go b/internal/config/pipeline_composition.go index fc95102..b209fff 100644 --- a/internal/config/pipeline_composition.go +++ b/internal/config/pipeline_composition.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "unicode" "gitea.maximumdirect.net/eric/narratio/internal/fileops" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" @@ -12,10 +13,18 @@ import ( ) type pipelineResolutionMetadata struct { - rootPath string - imports []string - sources []string - ownership []pipelineFieldOwnership + rootPath string + imports []string + sources []string + selectedProfile *pipelineProfileSelection + effectiveDigest string + ownership []pipelineFieldOwnership +} + +type pipelineProfileSelection struct { + name string + source string + overlayPath string } type pipelineFieldOwnership struct { @@ -24,10 +33,17 @@ type pipelineFieldOwnership struct { } type pipelineCompositionEnvelope struct { - imports []string + imports []string + defaultProfile *string + profiles []pipelineProfileDeclaration } -func loadComposedPipeline(path string) (*PipelineConfig, error) { +type pipelineProfileDeclaration struct { + name string + overlay string +} + +func loadComposedPipeline(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { rootPath, err := filepath.Abs(path) if err != nil { return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err) @@ -53,6 +69,14 @@ func loadComposedPipeline(path string) (*PipelineConfig, error) { if err != nil { return nil, err } + selection, err := selectPipelineProfile(envelope, opts) + if err != nil { + return nil, err + } + overlays, err := loadPipelineProfileOverlays(rootPath, envelope.profiles, imports) + if err != nil { + return nil, err + } documents := make([]*compositionDocument, 0, len(imports)+1) documents = append(documents, baseRoot) for _, imported := range imports { @@ -62,6 +86,17 @@ func loadComposedPipeline(path string) (*PipelineConfig, error) { if err != nil { return nil, err } + if selection != nil { + overlay, ok := loadedProfileOverlay(overlays, selection.name) + if !ok { + return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name) + } + merged, err = mergeOverlayComposition(merged, overlay.document) + if err != nil { + return nil, err + } + selection.overlayPath = overlay.path + } rendered, err := merged.canonicalYAML() if err != nil { @@ -76,8 +111,9 @@ func loadComposedPipeline(path string) (*PipelineConfig, error) { return nil, err } metadata := &pipelineResolutionMetadata{ - rootPath: rootPath, - sources: append([]string(nil), merged.sources...), + rootPath: rootPath, + sources: append([]string(nil), merged.sources...), + selectedProfile: selection, } for _, imported := range imports { metadata.imports = append(metadata.imports, imported.path) @@ -127,6 +163,39 @@ func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositi } envelope.imports = append(envelope.imports, item.value) } + case "default_profile": + if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { + return nil, pipelineCompositionEnvelope{}, fmt.Errorf( + "configuration source %s at composition.default_profile: expected a string profile name", + formatCompositionSources(field.value.sources), + ) + } + name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile") + if err != nil { + return nil, pipelineCompositionEnvelope{}, err + } + envelope.defaultProfile = &name + case "profiles": + if field.value.kind != yaml.MappingNode { + return nil, pipelineCompositionEnvelope{}, fmt.Errorf( + "configuration source %s at composition.profiles: expected a mapping, got %s", + formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), + ) + } + if len(field.value.fields) == 0 { + return nil, pipelineCompositionEnvelope{}, fmt.Errorf("composition.profiles must declare at least one named profile") + } + for _, profileField := range field.value.fields { + name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name") + if err != nil { + return nil, pipelineCompositionEnvelope{}, err + } + profile, err := parsePipelineProfileDeclaration(name, profileField.value) + if err != nil { + return nil, pipelineCompositionEnvelope{}, err + } + envelope.profiles = append(envelope.profiles, profile) + } default: return nil, pipelineCompositionEnvelope{}, fmt.Errorf( "configuration source %s at composition.%s: unknown composition field %q", @@ -134,6 +203,11 @@ func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositi ) } } + if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) { + return nil, pipelineCompositionEnvelope{}, fmt.Errorf( + "composition.default_profile %q does not name a declared profile", *envelope.defaultProfile, + ) + } root.fields = append(root.fields[:index], root.fields[index+1:]...) for fieldIndex := range root.fields { root.fields[fieldIndex].order = fieldIndex @@ -141,12 +215,179 @@ func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositi return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}, envelope, nil } +func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) { + path := "composition.profiles." + name + if node.kind != yaml.MappingNode { + return pipelineProfileDeclaration{}, fmt.Errorf( + "configuration source %s at %s: expected a mapping, got %s", + formatCompositionSources(node.sources), path, yamlKindName(node.kind), + ) + } + profile := pipelineProfileDeclaration{name: name} + for _, field := range node.fields { + if field.key != "overlay" { + return pipelineProfileDeclaration{}, fmt.Errorf( + "configuration source %s at %s.%s: unknown profile field %q; only overlay is supported", + formatCompositionSources(field.value.sources), path, field.key, field.key, + ) + } + if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { + return pipelineProfileDeclaration{}, fmt.Errorf( + "configuration source %s at %s.overlay: expected a string path", + formatCompositionSources(field.value.sources), path, + ) + } + profile.overlay = field.value.value + } + if profile.overlay == "" { + return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path) + } + return profile, nil +} + +func normalizePipelineProfileName(value, label string) (string, error) { + if value == "" || strings.TrimSpace(value) != value { + return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return "", fmt.Errorf("%s %q must not contain control characters", label, value) + } + } + return value, nil +} + +func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool { + for _, profile := range profiles { + if profile.name == name { + return true + } + } + return false +} + +func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) { + if opts.Profile != nil { + name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection") + if err != nil { + return nil, err + } + if len(envelope.profiles) == 0 { + return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name) + } + if !pipelineProfileDeclared(envelope.profiles, name) { + return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name) + } + return &pipelineProfileSelection{name: name, source: "cli"}, nil + } + if len(envelope.profiles) == 0 { + return nil, nil + } + if envelope.defaultProfile == nil { + return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected") + } + return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil +} + type loadedPipelineImport struct { path string document *compositionDocument info os.FileInfo } +type loadedPipelineProfileOverlay struct { + profile string + path string + document *compositionDocument + info os.FileInfo +} + +func loadPipelineProfileOverlays( + rootPath string, + declared []pipelineProfileDeclaration, + imports []loadedPipelineImport, +) ([]loadedPipelineProfileOverlay, error) { + if len(declared) == 0 { + return nil, nil + } + rootDir := filepath.Dir(rootPath) + rootInfo, err := os.Stat(rootPath) + if err != nil { + return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) + } + seenPaths := make(map[string]string, len(declared)) + loaded := make([]loadedPipelineProfileOverlay, 0, len(declared)) + for _, profile := range declared { + label := "composition.profiles." + profile.name + ".overlay" + raw := profile.overlay + if strings.TrimSpace(raw) != raw || raw == "" { + return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label) + } + normalized, err := pathsafe.NormalizeRelativeDestination(raw) + if err != nil { + return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err) + } + extension := filepath.Ext(filepath.FromSlash(normalized)) + if extension != ".yml" && extension != ".yaml" { + return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw) + } + if prior, duplicate := seenPaths[normalized]; duplicate { + return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior) + } + seenPaths[normalized] = profile.name + resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) + file, err := fileops.OpenConfinedRegularFile(rootDir, normalized) + if err != nil { + return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err) + } + info, statErr := file.Stat() + if statErr != nil { + _ = file.Close() + return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr) + } + if os.SameFile(rootInfo, info) { + _ = file.Close() + return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw) + } + for _, imported := range imports { + if os.SameFile(imported.info, info) { + _ = file.Close() + return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path) + } + } + for _, prior := range loaded { + if os.SameFile(prior.info, info) { + _ = file.Close() + return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile) + } + } + document, parseErr := parseCompositionDocument(resolved, file) + closeErr := file.Close() + if parseErr != nil { + return nil, parseErr + } + if closeErr != nil { + return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr) + } + if compositionFieldIndex(document.root.fields, "composition") >= 0 { + return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved) + } + loaded = append(loaded, loadedPipelineProfileOverlay{ + profile: profile.name, path: resolved, document: document, info: info, + }) + } + return loaded, nil +} + +func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) { + for _, overlay := range overlays { + if overlay.profile == name { + return overlay, true + } + } + return loadedPipelineProfileOverlay{}, false +} + func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) { if len(declared) == 0 { return nil, nil diff --git a/internal/config/pipeline_composition_test.go b/internal/config/pipeline_composition_test.go index b819284..97ee9a4 100644 --- a/internal/config/pipeline_composition_test.go +++ b/internal/config/pipeline_composition_test.go @@ -237,12 +237,12 @@ func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t * } }) - t.Run("future profile field", func(t *testing.T) { + t.Run("empty profiles", func(t *testing.T) { dir := t.TempDir() rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n") _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), "unknown composition field") || !strings.Contains(err.Error(), "profiles") { - t.Fatalf("LoadPipeline() error = %v, want profile field rejected before its implementation", err) + if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") { + t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err) } }) } diff --git a/internal/config/pipeline_profiles_test.go b/internal/config/pipeline_profiles_test.go new file mode 100644 index 0000000..2b61d90 --- /dev/null +++ b/internal/config/pipeline_profiles_test.go @@ -0,0 +1,392 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, `composition: + imports: [conf.d/base.yml] + default_profile: production + profiles: + production: + overlay: profiles/production.yml + testing: + overlay: profiles/testing.yml +whisperx: + transcribe_url: https://transcription.example.com/transcribe +`) + importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace: + root: /srv/base +audita: + modules: [base, shared] +`) + productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace: + root: /srv/production +whisperx: + language: en +`) + testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace: + root: /srv/testing +whisperx: + language: fr +`) + + production, err := LoadPipeline(rootPath) + if err != nil { + t.Fatal(err) + } + if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" { + t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language) + } + assertSelectedPipelineProfile(t, production, "production", "default", productionPath) + if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !reflect.DeepEqual(production.resolution.sources, want) { + t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want) + } + assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath)) + assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath)) + assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource) + + profile := "testing" + testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile}) + if err != nil { + t.Fatal(err) + } + if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" { + t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language) + } + assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath) +} + +func TestLoadPipelineProfileSelectionErrors(t *testing.T) { + for _, test := range []struct { + name string + root string + profile *string + want string + }{ + {name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"}, + {name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"}, + {name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"}, + {name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"}, + {name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"}, + {name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"}, + {name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"}, + {name: "empty profile name", root: `composition: + default_profile: production + profiles: + "": + overlay: production.yml +whisperx: + transcribe_url: https://transcription.example.com/transcribe +`, want: "profile name must be non-empty"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + root := test.root + if test.name == "empty default" { + root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`) + } + rootPath := writeProfilePipeline(t, dir, root) + writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n") + _, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { + t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want) + } + }) + } + + t.Run("explicit against profile-free pipeline", func(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML) + selected := "testing" + _, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected}) + if err == nil || !strings.Contains(err.Error(), "declares no profiles") { + t.Fatalf("error = %v, want profile-free rejection", err) + } + }) +} + +func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) { + if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") { + t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err) + } +} + +func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) { + for _, test := range []struct { + name string + path string + content string + setup func(*testing.T, string) + want string + }{ + {name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"}, + {name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"}, + {name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"}, + {name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"}, + {name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"}, + {name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) { + rootPath := filepath.Join(dir, "pipeline.yml") + data, err := os.ReadFile(rootPath) + if err != nil { + t.Fatal(err) + } + updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1) + if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil { + t.Fatal(err) + } + }, want: "only overlay is supported"}, + {name: "traversal", path: "../testing.yml", want: "invalid"}, + {name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"}, + {name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) { + if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil { + t.Fatal(err) + } + }, want: "regular file"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, `composition: + default_profile: production + profiles: + production: + overlay: profiles/production.yml + testing: + overlay: `+test.path+` +whisperx: + transcribe_url: https://transcription.example.com/transcribe +`) + writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n") + if test.content != "" { + writePipelineSource(t, dir, test.path, test.content) + } + if test.setup != nil { + test.setup(t, dir) + } + _, err := LoadPipeline(rootPath) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { + t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want) + } + }) + } +} + +func TestLoadPipelineProfileOverlaySemantics(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, `composition: + default_profile: testing + profiles: + testing: + overlay: testing.yml +workspace: + root: /base + cleanup_after_publish: true +whisperx: + transcribe_url: https://transcription.example.com/transcribe + retries: 3 +audita: + modules: [base, shared] +scriptorium: + artifacts: + session_recap: + enabled: true + prompt_id: dnd.session_recap + output_path: artifacts/session_recap.md +`) + writePipelineSource(t, dir, "testing.yml", `workspace: + cleanup_after_publish: false +whisperx: + retries: 0 +audita: + modules: [] +scriptorium: + artifacts: + session_recap: + enabled: false + experimental: + enabled: false +`) + + cfg, err := LoadPipeline(rootPath) + if err != nil { + t.Fatal(err) + } + if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 { + t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries) + } + if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 { + t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules) + } + if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 { + t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium) + } +} + +func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) { + for _, test := range []struct { + name string + overlay string + want string + }{ + {name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"}, + {name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"}, + {name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, `composition: + default_profile: testing + profiles: + testing: + overlay: testing.yml +workspace: + root: /base +audita: + modules: [base] +whisperx: + transcribe_url: https://transcription.example.com/transcribe +`) + writePipelineSource(t, dir, "testing.yml", test.overlay) + _, err := LoadPipeline(rootPath) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { + t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want) + } + }) + } +} + +func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) { + dir := t.TempDir() + monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace: + root: /srv/narratio +whisperx: + transcribe_url: https://transcription.example.com/transcribe + language: en +`) + composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition: + imports: [workspace.yml] +whisperx: + transcribe_url: https://transcription.example.com/transcribe + language: en +`) + writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n") + + monolithic, err := LoadPipeline(monolithicPath) + if err != nil { + t.Fatal(err) + } + composed, err := LoadPipeline(composedPath) + if err != nil { + t.Fatal(err) + } + if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest { + t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest) + } + + changedPath := writePipelineSource(t, dir, "changed.yml", `workspace: + root: /srv/narratio +whisperx: + transcribe_url: https://transcription.example.com/transcribe + language: fr +`) + changed, err := LoadPipeline(changedPath) + if err != nil { + t.Fatal(err) + } + if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest { + t.Fatal("semantic value change retained effective digest") + } + + t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one") + first, err := LoadPipeline(monolithicPath) + if err != nil { + t.Fatal(err) + } + t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two") + second, err := LoadPipeline(monolithicPath) + if err != nil { + t.Fatal(err) + } + if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") { + t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest) + } +} + +func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing")) + overlay := "workspace:\n root: /srv/narratio\n" + writePipelineSource(t, dir, "production.yml", overlay) + writePipelineSource(t, dir, "testing.yml", overlay) + + production := "production" + productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production}) + if err != nil { + t.Fatal(err) + } + testing := "testing" + testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing}) + if err != nil { + t.Fatal(err) + } + if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest { + t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest) + } +} + +func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) { + dir := t.TempDir() + rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing")) + writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n") + writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n") + campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\n") + sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n") + selected := "testing" + cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected}) + if err != nil { + t.Fatal(err) + } + if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" { + t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile) + } +} + +func writeProfilePipeline(t *testing.T, dir, content string) string { + t.Helper() + return writePipelineSource(t, dir, "pipeline.yml", content) +} + +func profileComposition(defaultProfile string, profiles ...string) string { + var builder strings.Builder + builder.WriteString("composition:\n") + if defaultProfile != "" { + builder.WriteString(" default_profile: " + defaultProfile + "\n") + } + builder.WriteString(" profiles:\n") + for _, profile := range profiles { + builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n") + } + builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n") + return builder.String() +} + +func profilePointer(value string) *string { + return &value +} + +func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) { + t.Helper() + if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil { + t.Fatal("selected profile provenance is absent") + } + selection := cfg.resolution.selectedProfile + if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) { + t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath)) + } +}